{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "fa816164",
   "metadata": {},
   "source": [
    "## 数据准备"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "48a4d118",
   "metadata": {},
   "source": [
    "### 代码包引入"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "783839ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.utils.data as Data\n",
    "import numpy as np\n",
    "from torch import optim\n",
    "import random\n",
    "from tqdm import *\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ddfa59fa",
   "metadata": {},
   "source": [
    "### 数据集生成"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5a0ee89f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([['kju:', 'kei', 'em', 'i:', 'vi:', 'pi:'],\n",
       "  ['bi:', 'kju:', 'eit∫', 'eks', 'ef', 'di:']],\n",
       " [['q', 'k', 'm', 'e', 'v', 'p'], ['b', 'q', 'h', 'x', 'f', 'd']])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 数据集生成\n",
    "soundmark = ['ei',  'bi:',  'si:',  'di:',  'i:',  'ef',  'dʒi:',  'eit∫',  'ai', 'dʒei', 'kei', 'el', 'em', 'en', 'əu', 'pi:', 'kju:',\n",
    "        'ɑ:', 'es', 'ti:', 'ju:', 'vi:', 'd∧blju:', 'eks', 'wai', 'zi:']\n",
    "\n",
    "alphabet = ['a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q',\n",
    "         'r','s','t','u','v','w','x','y','z']\n",
    "\n",
    "t = 1000 #总条数\n",
    "r = 0.9   #扰动项\n",
    "seq_len = 6\n",
    "src_tokens, tgt_tokens = [],[] #原始序列、目标序列列表\n",
    "\n",
    "for i in range(t):\n",
    "    src, tgt = [],[]\n",
    "    for j in range(seq_len):\n",
    "        ind = random.randint(0,25)\n",
    "        src.append(soundmark[ind])\n",
    "        if random.random() < r:\n",
    "            tgt.append(alphabet[ind])\n",
    "        else:\n",
    "            tgt.append(alphabet[random.randint(0,25)])\n",
    "    src_tokens.append(src)\n",
    "    tgt_tokens.append(tgt)\n",
    "src_tokens[:2], tgt_tokens[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f99844b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter  # 计数类\n",
    "\n",
    "flatten = lambda l: [item for sublist in l for item in sublist]  # 展平数组\n",
    "# 构建词表\n",
    "class Vocab:\n",
    "    def __init__(self, tokens):\n",
    "        self.tokens = tokens  # 传入的tokens是二维列表\n",
    "        self.token2index = {'<pad>': 0, '<bos>': 1, '<eos>': 2, '<unk>': 3}  # 先存好特殊词元\n",
    "        # 将词元按词频排序后生成列表\n",
    "        self.token2index.update({\n",
    "            token: index + 4\n",
    "            for index, (token, freq) in enumerate(\n",
    "                sorted(Counter(flatten(self.tokens)).items(), key=lambda x: x[1], reverse=True))\n",
    "        })\n",
    "        # 构建id到词元字典\n",
    "        self.index2token = {index: token for token, index in self.token2index.items()}\n",
    "\n",
    "    def __getitem__(self, query):\n",
    "        # 单一索引\n",
    "        if isinstance(query, (str, int)):\n",
    "            if isinstance(query, str):\n",
    "                return self.token2index.get(query, 3)\n",
    "            elif isinstance(query, (int)):\n",
    "                return self.index2token.get(query, '<unk>')\n",
    "        # 数组索引\n",
    "        elif isinstance(query, (list, tuple)):\n",
    "            return [self.__getitem__(item) for item in query]\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.index2token)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01ad7798",
   "metadata": {},
   "source": [
    "### 数据集构造"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "246e630e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "\n",
    "#实例化source和target词表\n",
    "src_vocab, tgt_vocab = Vocab(src_tokens), Vocab(tgt_tokens)\n",
    "src_vocab_size = len(src_vocab)  # 源语言词表大小\n",
    "tgt_vocab_size = len(tgt_vocab)  # 目标语言词表大小\n",
    "\n",
    "#增加开始标识<bos>和结尾标识<eos>\n",
    "encoder_input = torch.tensor([src_vocab[line + ['<pad>']] for line in src_tokens])\n",
    "decoder_input = torch.tensor([tgt_vocab[['<bos>'] + line] for line in tgt_tokens])\n",
    "decoder_output = torch.tensor([tgt_vocab[line + ['<eos>']] for line in tgt_tokens])\n",
    "\n",
    "# 训练集和测试集比例8比2，batch_size = 16\n",
    "train_size = int(len(encoder_input) * 0.8)\n",
    "test_size = len(encoder_input) - train_size\n",
    "batch_size = 16\n",
    "\n",
    "# 自定义数据集函数\n",
    "class MyDataSet(Data.Dataset):\n",
    "    def __init__(self, enc_inputs, dec_inputs, dec_outputs):\n",
    "        super(MyDataSet, self).__init__()\n",
    "        self.enc_inputs = enc_inputs\n",
    "        self.dec_inputs = dec_inputs\n",
    "        self.dec_outputs = dec_outputs\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.enc_inputs.shape[0]\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        return self.enc_inputs[idx], self.dec_inputs[idx], self.dec_outputs[idx]\n",
    "\n",
    "train_loader = DataLoader(MyDataSet(encoder_input[:train_size], decoder_input[:train_size], decoder_output[:train_size]), batch_size=batch_size)\n",
    "test_loader = DataLoader(MyDataSet(encoder_input[-test_size:], decoder_input[-test_size:], decoder_output[-test_size:]), batch_size=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "040214ea",
   "metadata": {},
   "source": [
    "## 模型构建"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "54dddf8b",
   "metadata": {},
   "source": [
    "### 位置编码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8ad8a03f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_sinusoid_encoding_table(n_position, d_model):\n",
    "    def cal_angle(position, hid_idx):\n",
    "        return position / np.power(10000, 2 * (hid_idx // 2) / d_model)\n",
    "    def get_posi_angle_vec(position):\n",
    "        return [cal_angle(position, hid_j) for hid_j in range(d_model)]\n",
    "    sinusoid_table = np.array([get_posi_angle_vec(pos_i) for pos_i in range(n_position)])\n",
    "    sinusoid_table[:, 0::2] = np.sin(sinusoid_table[:, 0::2]) # 偶数位用正弦函数\n",
    "    sinusoid_table[:, 1::2] = np.cos(sinusoid_table[:, 1::2]) # 奇数位用余弦函数\n",
    "    return torch.FloatTensor(sinusoid_table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "73ed603c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 0.0000e+00,  1.0000e+00,  0.0000e+00,  ...,  1.0000e+00,\n",
      "          0.0000e+00,  1.0000e+00],\n",
      "        [ 8.4147e-01,  5.4030e-01,  8.2186e-01,  ...,  1.0000e+00,\n",
      "          1.0366e-04,  1.0000e+00],\n",
      "        [ 9.0930e-01, -4.1615e-01,  9.3641e-01,  ...,  1.0000e+00,\n",
      "          2.0733e-04,  1.0000e+00],\n",
      "        ...,\n",
      "        [ 9.5638e-01, -2.9214e-01,  7.9142e-01,  ...,  1.0000e+00,\n",
      "          2.7989e-03,  1.0000e+00],\n",
      "        [ 2.7091e-01, -9.6261e-01,  9.5325e-01,  ...,  1.0000e+00,\n",
      "          2.9026e-03,  1.0000e+00],\n",
      "        [-6.6363e-01, -7.4806e-01,  2.9471e-01,  ...,  1.0000e+00,\n",
      "          3.0062e-03,  1.0000e+00]])\n"
     ]
    }
   ],
   "source": [
    "print(get_sinusoid_encoding_table(30, 512))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "77f721b9",
   "metadata": {},
   "source": [
    "### 掩码操作"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "16f45380",
   "metadata": {},
   "outputs": [],
   "source": [
    "# mask掉没有意义的占位符\n",
    "def get_attn_pad_mask(seq_q, seq_k):                       # seq_q: [batch_size, seq_len] ,seq_k: [batch_size, seq_len]\n",
    "    batch_size, len_q = seq_q.size()\n",
    "    batch_size, len_k = seq_k.size()\n",
    "    pad_attn_mask = seq_k.data.eq(0).unsqueeze(1)          # 判断 输入那些含有P(=0),用1标记 ,[batch_size, 1, len_k]\n",
    "    return pad_attn_mask.expand(batch_size, len_q, len_k)\n",
    "\n",
    "# mask掉未来信息\n",
    "def get_attn_subsequence_mask(seq):                               # seq: [batch_size, tgt_len]\n",
    "    attn_shape = [seq.size(0), seq.size(1), seq.size(1)]\n",
    "    subsequence_mask = np.triu(np.ones(attn_shape), k=1)          # 生成上三角矩阵,[batch_size, tgt_len, tgt_len]\n",
    "    subsequence_mask = torch.from_numpy(subsequence_mask).byte()  #  [batch_size, tgt_len, tgt_len]\n",
    "    return subsequence_mask "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1da18369",
   "metadata": {},
   "source": [
    "### 注意力计算函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0e7fb2ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 缩放点积注意力计算\n",
    "class ScaledDotProductAttention(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(ScaledDotProductAttention, self).__init__()\n",
    "    def forward(self, Q, K, V, attn_mask):\n",
    "        '''\n",
    "        Q: [batch_size, n_heads, len_q, d_k]\n",
    "        K: [batch_size, n_heads, len_k, d_k]\n",
    "        V: [batch_size, n_heads, len_v(=len_k), d_v]\n",
    "        attn_mask: [batch_size, n_heads, seq_len, seq_len]\n",
    "        '''\n",
    "        scores = torch.matmul(Q, K.transpose(-1, -2)) / np.sqrt(d_k) # scores : [batch_size, n_heads, len_q, len_k]\n",
    "        scores.masked_fill_(attn_mask, -1e9) # Fills elements of self tensor with value where mask is True.\n",
    "        attn = nn.Softmax(dim=-1)(scores)\n",
    "        context = torch.matmul(attn, V) # [batch_size, n_heads, len_q, d_v]\n",
    "        return context, attn\n",
    "\n",
    "#多头注意力计算\n",
    "class MultiHeadAttention(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(MultiHeadAttention, self).__init__()\n",
    "        self.W_Q = nn.Linear(d_model, d_k * n_heads, bias=False)\n",
    "        self.W_K = nn.Linear(d_model, d_k * n_heads, bias=False)\n",
    "        self.W_V = nn.Linear(d_model, d_v * n_heads, bias=False)\n",
    "        self.fc = nn.Linear(n_heads * d_v, d_model, bias=False)\n",
    "    def forward(self, input_Q, input_K, input_V, attn_mask):\n",
    "        '''\n",
    "        input_Q: [batch_size, len_q, d_model]\n",
    "        input_K: [batch_size, len_k, d_model]\n",
    "        input_V: [batch_size, len_v(=len_k), d_model]\n",
    "        attn_mask: [batch_size, seq_len, seq_len]\n",
    "        '''\n",
    "        residual, batch_size = input_Q, input_Q.size(0)\n",
    "        # (B, S, D) -proj-> (B, S, D_new) -split-> (B, S, H, W) -trans-> (B, H, S, W)\n",
    "        Q = self.W_Q(input_Q).view(batch_size, -1, n_heads, d_k).transpose(1,2) # Q: [batch_size, n_heads, len_q, d_k]\n",
    "        K = self.W_K(input_K).view(batch_size, -1, n_heads, d_k).transpose(1,2) # K: [batch_size, n_heads, len_k, d_k]\n",
    "        V = self.W_V(input_V).view(batch_size, -1, n_heads, d_v).transpose(1,2) # V: [batch_size, n_heads, len_v(=len_k), d_v]\n",
    "        attn_mask = attn_mask.unsqueeze(1).repeat(1, n_heads, 1, 1) # attn_mask : [batch_size, n_heads, seq_len, seq_len]\n",
    "        # context: [batch_size, n_heads, len_q, d_v], attn: [batch_size, n_heads, len_q, len_k]\n",
    "        context, attn = ScaledDotProductAttention()(Q, K, V, attn_mask)\n",
    "        context = context.transpose(1, 2).reshape(batch_size, -1, n_heads * d_v) # context: [batch_size, len_q, n_heads * d_v]\n",
    "        output = self.fc(context) # [batch_size, len_q, d_model]\n",
    "        return nn.LayerNorm(d_model)(output + residual), attn"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "852dc93f",
   "metadata": {},
   "source": [
    "### 构建前馈网络"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b68eb3bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "class PoswiseFeedForwardNet(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(PoswiseFeedForwardNet, self).__init__()\n",
    "        self.fc = nn.Sequential(\n",
    "            nn.Linear(d_model, d_ff, bias=False),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(d_ff, d_model, bias=False))\n",
    "        \n",
    "    def forward(self, inputs):                             # inputs: [batch_size, seq_len, d_model]\n",
    "        residual = inputs\n",
    "        output = self.fc(inputs)\n",
    "        return nn.LayerNorm(d_model)(output + residual)   # 残差 + LayerNorm"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "44b3ff96",
   "metadata": {},
   "source": [
    "### 编码器模块"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "eae1e01c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 编码器层\n",
    "class EncoderLayer(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(EncoderLayer, self).__init__()\n",
    "        self.enc_self_attn = MultiHeadAttention()  # 多头注意力\n",
    "        self.pos_ffn = PoswiseFeedForwardNet()  # 前馈网络\n",
    "    def forward(self, enc_inputs, enc_self_attn_mask):\n",
    "        '''\n",
    "        enc_inputs: [batch_size, src_len, d_model]\n",
    "        enc_self_attn_mask: [batch_size, src_len, src_len]\n",
    "        '''\n",
    "        # enc_outputs: [batch_size, src_len, d_model], attn: [batch_size, n_heads, src_len, src_len]\n",
    "        enc_outputs, attn = self.enc_self_attn(enc_inputs, enc_inputs, enc_inputs, enc_self_attn_mask) # enc_inputs to same Q,K,V\n",
    "        enc_outputs = self.pos_ffn(enc_outputs) # enc_outputs: [batch_size, src_len, d_model]\n",
    "        return enc_outputs, attn\n",
    "\n",
    "# 编码器模块\n",
    "class Encoder(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Encoder, self).__init__()\n",
    "        self.src_emb = nn.Embedding(src_vocab_size, d_model)\n",
    "        self.pos_emb = nn.Embedding.from_pretrained(get_sinusoid_encoding_table(src_vocab_size, d_model), freeze=True)\n",
    "        self.layers = nn.ModuleList([EncoderLayer() for _ in range(n_layers)])\n",
    "    def forward(self, enc_inputs):\n",
    "        '''\n",
    "        enc_inputs: [batch_size, src_len]\n",
    "        '''\n",
    "        word_emb = self.src_emb(enc_inputs) # [batch_size, src_len, d_model]\n",
    "        pos_emb = self.pos_emb(enc_inputs) # [batch_size, src_len, d_model]\n",
    "        enc_outputs = word_emb + pos_emb\n",
    "        enc_self_attn_mask = get_attn_pad_mask(enc_inputs, enc_inputs) # [batch_size, src_len, src_len]\n",
    "        enc_self_attns = []\n",
    "        for layer in self.layers:\n",
    "            # enc_outputs: [batch_size, src_len, d_model], enc_self_attn: [batch_size, n_heads, src_len, src_len]\n",
    "            enc_outputs, enc_self_attn = layer(enc_outputs, enc_self_attn_mask)\n",
    "            enc_self_attns.append(enc_self_attn)\n",
    "        return enc_outputs, enc_self_attns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a62a13bc",
   "metadata": {},
   "source": [
    "### 解码器模块"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "416d9b38",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 解码器层\n",
    "class DecoderLayer(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(DecoderLayer, self).__init__()\n",
    "        self.dec_self_attn = MultiHeadAttention()\n",
    "        self.dec_enc_attn = MultiHeadAttention()\n",
    "        self.pos_ffn = PoswiseFeedForwardNet()\n",
    "    def forward(self, dec_inputs, enc_outputs, dec_self_attn_mask, dec_enc_attn_mask):\n",
    "        '''\n",
    "        dec_inputs: [batch_size, tgt_len, d_model]\n",
    "        enc_outputs: [batch_size, src_len, d_model]\n",
    "        dec_self_attn_mask: [batch_size, tgt_len, tgt_len]\n",
    "        dec_enc_attn_mask: [batch_size, tgt_len, src_len]\n",
    "        '''\n",
    "        # dec_outputs: [batch_size, tgt_len, d_model], dec_self_attn: [batch_size, n_heads, tgt_len, tgt_len]\n",
    "        dec_outputs, dec_self_attn = self.dec_self_attn(dec_inputs, dec_inputs, dec_inputs, dec_self_attn_mask)\n",
    "        # dec_outputs: [batch_size, tgt_len, d_model], dec_enc_attn: [batch_size, h_heads, tgt_len, src_len]\n",
    "        dec_outputs, dec_enc_attn = self.dec_enc_attn(dec_outputs, enc_outputs, enc_outputs, dec_enc_attn_mask)\n",
    "        dec_outputs = self.pos_ffn(dec_outputs) # [batch_size, tgt_len, d_model]\n",
    "        return dec_outputs, dec_self_attn, dec_enc_attn\n",
    "\n",
    "# 解码器模块\n",
    "class Decoder(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Decoder, self).__init__()\n",
    "        self.tgt_emb = nn.Embedding(tgt_vocab_size, d_model)\n",
    "        self.pos_emb = nn.Embedding.from_pretrained(get_sinusoid_encoding_table(tgt_vocab_size, d_model),freeze=True)\n",
    "        self.layers = nn.ModuleList([DecoderLayer() for _ in range(n_layers)])\n",
    "    def forward(self, dec_inputs, enc_inputs, enc_outputs):\n",
    "        '''\n",
    "        dec_inputs: [batch_size, tgt_len]\n",
    "        enc_intpus: [batch_size, src_len]\n",
    "        enc_outputs: [batsh_size, src_len, d_model]\n",
    "        '''\n",
    "        word_emb = self.tgt_emb(dec_inputs) # [batch_size, tgt_len, d_model]\n",
    "        pos_emb = self.pos_emb(dec_inputs) # [batch_size, tgt_len, d_model]\n",
    "        dec_outputs = word_emb + pos_emb\n",
    "        dec_self_attn_pad_mask = get_attn_pad_mask(dec_inputs, dec_inputs) # [batch_size, tgt_len, tgt_len]\n",
    "        dec_self_attn_subsequent_mask = get_attn_subsequence_mask(dec_inputs) # [batch_size, tgt_len]\n",
    "        dec_self_attn_mask = torch.gt((dec_self_attn_pad_mask + dec_self_attn_subsequent_mask), 0) # [batch_size, tgt_len, tgt_len]\n",
    "        dec_enc_attn_mask = get_attn_pad_mask(dec_inputs, enc_inputs) # [batc_size, tgt_len, src_len]\n",
    "        dec_self_attns, dec_enc_attns = [], []\n",
    "        for layer in self.layers:\n",
    "            # dec_outputs: [batch_size, tgt_len, d_model], dec_self_attn: [batch_size, n_heads, tgt_len, tgt_len], dec_enc_attn: [batch_size, h_heads, tgt_len,src_len]\n",
    "            dec_outputs, dec_self_attn, dec_enc_attn = layer(dec_outputs, enc_outputs, dec_self_attn_mask, dec_enc_attn_mask)\n",
    "            dec_self_attns.append(dec_self_attn)\n",
    "            dec_enc_attns.append(dec_enc_attn)\n",
    "        return dec_outputs, dec_self_attns, dec_enc_attns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fe516394",
   "metadata": {},
   "source": [
    "### Transformer模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "5461ca2c",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Transformer(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Transformer, self).__init__()\n",
    "        self.encoder = Encoder()\n",
    "        self.decoder = Decoder()\n",
    "        self.projection = nn.Linear(d_model, tgt_vocab_size, bias=False)\n",
    "    def forward(self, enc_inputs, dec_inputs):\n",
    "        '''\n",
    "        enc_inputs: [batch_size, src_len]\n",
    "        dec_inputs: [batch_size, tgt_len]\n",
    "        '''\n",
    "        # tensor to store decoder outputs\n",
    "        # outputs = torch.zeros(batch_size, tgt_len, tgt_vocab_size).to(self.device)\n",
    "        # enc_outputs: [batch_size, src_len, d_model], enc_self_attns: [n_layers, batch_size, n_heads, src_len, src_len]\n",
    "        enc_outputs, enc_self_attns = self.encoder(enc_inputs)\n",
    "        # dec_outpus: [batch_size, tgt_len, d_model], dec_self_attns: [n_layers, batch_size, n_heads, tgt_len, tgt_len], dec_enc_attn: [n_layers, batch_size, tgt_len, src_len]\n",
    "        dec_outputs, dec_self_attns, dec_enc_attns = self.decoder(dec_inputs, enc_inputs, enc_outputs)\n",
    "        dec_logits = self.projection(dec_outputs) # dec_logits: [batch_size, tgt_len, tgt_vocab_size]\n",
    "        return dec_logits.view(-1, dec_logits.size(-1)), enc_self_attns, dec_self_attns, dec_enc_attns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c3328f5",
   "metadata": {},
   "source": [
    "## 模型训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6f2cf469",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|▏         | 1/50 [00:21<17:16, 21.15s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1 loss = 2.633762\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  4%|▍         | 2/50 [00:42<17:06, 21.39s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 2 loss = 2.063002\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  6%|▌         | 3/50 [01:04<16:48, 21.47s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 3 loss = 1.866944\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  8%|▊         | 4/50 [01:25<16:16, 21.23s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 4 loss = 1.802783\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 10%|█         | 5/50 [01:45<15:49, 21.10s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 5 loss = 1.643217\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 12%|█▏        | 6/50 [02:07<15:27, 21.07s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 6 loss = 1.803471\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 14%|█▍        | 7/50 [02:27<15:04, 21.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 7 loss = 1.518794\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 16%|█▌        | 8/50 [02:48<14:41, 20.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 8 loss = 1.632840\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 18%|█▊        | 9/50 [03:10<14:23, 21.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 9 loss = 1.446730\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 20%|██        | 10/50 [03:31<14:02, 21.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 10 loss = 1.340348\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 22%|██▏       | 11/50 [03:52<13:40, 21.04s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 11 loss = 1.366917\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 24%|██▍       | 12/50 [04:13<13:20, 21.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 12 loss = 1.499715\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 26%|██▌       | 13/50 [04:34<13:01, 21.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 13 loss = 1.371446\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 28%|██▊       | 14/50 [04:55<12:41, 21.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 14 loss = 1.380498\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 30%|███       | 15/50 [05:16<12:19, 21.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 15 loss = 1.298183\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 32%|███▏      | 16/50 [05:37<11:53, 20.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 16 loss = 1.107512\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 34%|███▍      | 17/50 [05:57<11:27, 20.85s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 17 loss = 1.015355\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 36%|███▌      | 18/50 [06:18<11:04, 20.76s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 18 loss = 0.891573\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 38%|███▊      | 19/50 [06:39<10:41, 20.69s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 19 loss = 1.035157\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 40%|████      | 20/50 [06:59<10:19, 20.64s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 20 loss = 1.059943\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 42%|████▏     | 21/50 [07:20<09:58, 20.64s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 21 loss = 0.995347\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 44%|████▍     | 22/50 [07:40<09:38, 20.65s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 22 loss = 0.828730\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 46%|████▌     | 23/50 [08:01<09:18, 20.68s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 23 loss = 0.717403\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 48%|████▊     | 24/50 [08:22<08:59, 20.77s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 24 loss = 0.768870\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 50%|█████     | 25/50 [08:43<08:39, 20.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 25 loss = 0.713927\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 52%|█████▏    | 26/50 [09:04<08:18, 20.75s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 26 loss = 0.797918\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 54%|█████▍    | 27/50 [09:24<07:57, 20.74s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 27 loss = 0.680246\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 56%|█████▌    | 28/50 [09:45<07:36, 20.76s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 28 loss = 0.611770\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 58%|█████▊    | 29/50 [10:06<07:16, 20.77s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 29 loss = 0.810355\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 60%|██████    | 30/50 [10:27<06:57, 20.86s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 30 loss = 0.537487\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 62%|██████▏   | 31/50 [10:48<06:37, 20.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 31 loss = 0.484650\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 64%|██████▍   | 32/50 [11:09<06:15, 20.86s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 32 loss = 0.447033\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 66%|██████▌   | 33/50 [11:30<05:54, 20.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 33 loss = 0.399072\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 68%|██████▊   | 34/50 [11:51<05:34, 20.90s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 34 loss = 0.379649\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 70%|███████   | 35/50 [12:12<05:13, 20.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 35 loss = 0.270823\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 72%|███████▏  | 36/50 [12:32<04:52, 20.91s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 36 loss = 0.337878\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 74%|███████▍  | 37/50 [12:53<04:30, 20.81s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 37 loss = 0.235440\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 76%|███████▌  | 38/50 [13:14<04:09, 20.77s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 38 loss = 0.337393\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 78%|███████▊  | 39/50 [13:35<03:49, 20.85s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 39 loss = 0.260191\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 80%|████████  | 40/50 [13:56<03:28, 20.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 40 loss = 0.210084\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 82%|████████▏ | 41/50 [14:17<03:09, 21.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 41 loss = 0.168616\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 84%|████████▍ | 42/50 [14:38<02:47, 20.97s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 42 loss = 0.213607\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 86%|████████▌ | 43/50 [14:58<02:25, 20.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 43 loss = 0.110551\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 88%|████████▊ | 44/50 [15:19<02:04, 20.74s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 44 loss = 0.183562\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 90%|█████████ | 45/50 [15:39<01:43, 20.62s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 45 loss = 0.095172\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 92%|█████████▏| 46/50 [16:00<01:22, 20.57s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 46 loss = 0.132387\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 94%|█████████▍| 47/50 [16:20<01:01, 20.52s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 47 loss = 0.163805\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 96%|█████████▌| 48/50 [16:41<00:40, 20.49s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 48 loss = 0.152195\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 98%|█████████▊| 49/50 [17:01<00:20, 20.49s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 49 loss = 0.086681\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 50/50 [17:22<00:00, 20.84s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 50 loss = 0.085496\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "d_model = 512   # 字 Embedding 的维度\n",
    "d_ff = 2048     # 前向传播隐藏层维度\n",
    "d_k = d_v = 64  # K(=Q), V的维度 \n",
    "n_layers = 6    # 有多少个encoder和decoder\n",
    "n_heads = 8     # Multi-Head Attention设置为8\n",
    "num_epochs = 50 # 训练50轮\n",
    "# 记录损失变化\n",
    "loss_history = []\n",
    "\n",
    "model = Transformer()\n",
    "criterion = nn.CrossEntropyLoss(ignore_index=0)\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.99)\n",
    "\n",
    "for epoch in tqdm(range(num_epochs)):\n",
    "    total_loss = 0\n",
    "    for enc_inputs, dec_inputs, dec_outputs in train_loader:\n",
    "        '''\n",
    "        enc_inputs: [batch_size, src_len]\n",
    "        dec_inputs: [batch_size, tgt_len]\n",
    "        dec_outputs: [batch_size, tgt_len]\n",
    "        '''\n",
    "        # enc_inputs, dec_inputs, dec_outputs = enc_inputs.to(device), dec_inputs.to(device), dec_outputs.to(device)\n",
    "        # outputs: [batch_size * tgt_len, tgt_vocab_size]\n",
    "        outputs, enc_self_attns, dec_self_attns, dec_enc_attns = model(enc_inputs, dec_inputs)\n",
    "        loss = criterion(outputs, dec_outputs.view(-1))\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        total_loss += loss.item()\n",
    "    avg_loss = total_loss/len(train_loader)\n",
    "    loss_history.append(avg_loss)\n",
    "    print('Epoch:', '%d' % (epoch + 1), 'loss =', '{:.6f}'.format(avg_loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "6c7f102c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAGdCAYAAADuR1K7AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABfpElEQVR4nO3dd3gUVdsG8HvTNgmkGCANEhI6BAgQBEKXXhUrKgLWT5QiAhbs8qroKyoizYIoIsKrAURBBJQqoQQChA4SSAgphJJK2u58f4RsdnZnW7K7s9m9f9e117U7c2bm7BAyT855zjkKQRAEEBERETkJN7krQERERGRNDG6IiIjIqTC4ISIiIqfC4IaIiIicCoMbIiIicioMboiIiMipMLghIiIip8LghoiIiJyKh9wVsDe1Wo0rV67Az88PCoVC7uoQERGRGQRBQEFBAcLDw+HmZrxtxuWCmytXriAiIkLuahAREVENpKeno0mTJkbLuFxw4+fnB6Dy5vj7+8tcGyIiIjJHfn4+IiIiNM9xY1wuuKnqivL392dwQ0REVMeYk1LChGIiIiJyKgxuiIiIyKkwuCEiIiKnwuCGiIiInAqDGyIiInIqDG6IiIjIqTC4ISIiIqfC4IaIiIicCoMbIiIicioMboiIiMipMLghIiIip8LghoiIiJyKyy2caSt5t8rx+bZzECDg7dExcleHiIjIZbHlxkpKy1X49p9UfLf3otxVISIicmkMbqzE073yVgoCoFILMteGiIjIdTG4sRIPd4XmfblKLWNNiIiIXJuswc2SJUvQsWNH+Pv7w9/fH/Hx8fjjjz8Mlt+xYwcUCoXe6/Tp03astbSqlhuAwQ0REZGcZE0obtKkCT788EO0aNECAPD999/jnnvuQXJyMmJiDCflnjlzBv7+/prPjRo1snldTdEOborLVPDz9pSxNkRERK5L1pab0aNHY8SIEWjVqhVatWqF999/H/Xr18e+ffuMHhccHIzQ0FDNy93d3U41Nszdrbpb6qtdF2SsCRERkWtzmJwblUqF1atXo6ioCPHx8UbLdu7cGWFhYRg4cCC2b99utGxpaSny8/NFL1vbn3rN5tcgIiIiabIHNykpKahfvz6USiUmTZqEdevWoV27dpJlw8LC8NVXXyEhIQFr165F69atMXDgQOzatcvg+efOnYuAgADNKyIiwlZfhYiIiByAQhAEWcctl5WVIS0tDTdv3kRCQgK++eYb7Ny502CAo2v06NFQKBTYsGGD5P7S0lKUlpZqPufn5yMiIgJ5eXmivB1riHp1IwCgW1QQ/jfJeOsTERERmS8/Px8BAQFmPb9ln6HYy8tLk1DctWtXHDx4EJ9//jm+/PJLs47v0aMHVq5caXC/UqmEUqm0Sl3N1bNFA7tej4iIiKrJ3i2lSxAEUUuLKcnJyQgLC7Nhjcz3eM8oAECFipP4ERERyUXWlpvXXnsNw4cPR0REBAoKCrB69Wrs2LEDmzdvBgDMnj0bGRkZWLFiBQBg/vz5iIqKQkxMDMrKyrBy5UokJCQgISFBzq+hofSsjBVLylUy14SIiMh1yRrcZGdnY/z48cjMzERAQAA6duyIzZs3Y/DgwQCAzMxMpKWlacqXlZVh1qxZyMjIgI+PD2JiYrBx40aMGDFCrq8govSoHJJeWsFJ/IiIiOQie0KxvVmSkGSpxTvO47+bz+DBuCb4+MFYq56biIjIlVny/Ha4nJu6jC03RERE8mNwY0VKD+bcEBERyY3BjRV5e1a23Gw5mY0KLp5JREQkCwY3VlTVcgMAvxy6LGNNiIiIXBeDGyuqarkBgOx88+fqISIiIuthcGNF2i037ryzREREsuAj2Iq0W27c3XhriYiI5MAnsBWx5YaIiEh+fARbUVA9L817N4VCxpoQERG5LgY3VtTIr3r1cQY3RERE8mBwY0XaOTfFZRUy1oSIiMh1MbixsoFtggEA87acRU5Bicy1ISIicj0MbqzMw726O+pYep6MNSEiInJNDG6szMNNe8QU826IiIjsjcGNDX20+bTcVSAiInI5DG6sbPOJLM3701kFMtaEiIjINTG4sTKVWpC7CkRERC6NwQ0RERE5FQY3VvZEryi5q0BEROTSGNxY2SvD2shdBSIiIpfG4MbKtGcpJiIiIvtjcGNj1wpL5a4CERGRS2FwYwO+XtWtN9n5DG6IiIjsicGNDbw3pr3m/fWiMhlrQkRE5HoY3NhAhap6rpvsfC6eSUREZE8MbmygXK3WvC8oKZexJkRERK6HwY0NdIsK0rxXccJiIiIiu2JwYwMtQ/wQE+4PAFBpteIQERGR7TG4sZF2YZXBTQXXmiIiIrIrBjc24uGuAACoGdwQERHZFYMbG3FTVAY3bLkhIiKyLwY3NuLhVhncqBjcEBER2RWDGxtxd6u8tQxuiIiI7IvBjY24376zDG6IiIjsi8GNjVS13DDnhoiIyL4Y3NhIVc7NuZxCmWtCRETkWmQNbpYsWYKOHTvC398f/v7+iI+Pxx9//GH0mJ07dyIuLg7e3t5o1qwZli5daqfaWsb9dnCz6+xV/JB4Ud7KEBERuRBZg5smTZrgww8/RFJSEpKSkjBgwADcc889OHHihGT51NRUjBgxAn369EFycjJee+01TJs2DQkJCXauuWXe/FX6+xAREZH1ech58dGjR4s+v//++1iyZAn27duHmJgYvfJLly5FZGQk5s+fDwBo27YtkpKSMG/ePNx///32qLLZKrSWXQj195axJkRERK7FYXJuVCoVVq9ejaKiIsTHx0uWSUxMxJAhQ0Tbhg4diqSkJJSXS6++XVpaivz8fNHLHkZ1DNe8z8ovQWmFyi7XJSIicnWyBzcpKSmoX78+lEolJk2ahHXr1qFdu3aSZbOyshASEiLaFhISgoqKCuTm5koeM3fuXAQEBGheERERVv8OUtqG+eOtUdXfo/Ubm+1yXSIiIlcne3DTunVrHDlyBPv27cNzzz2HiRMn4uTJkwbLK24va1BFEATJ7VVmz56NvLw8zSs9Pd16lTehTZif3a5FRERElWTNuQEALy8vtGjRAgDQtWtXHDx4EJ9//jm+/PJLvbKhoaHIysoSbcvJyYGHhwcaNGggeX6lUgmlUmn9ipvB10v220tERORyZG+50SUIAkpLSyX3xcfHY+vWraJtW7ZsQdeuXeHp6WmP6lnE18td9Plw2g2ZakJEROQ6ZA1uXnvtNezevRsXL15ESkoKXn/9dezYsQPjxo0DUNmlNGHCBE35SZMm4dKlS5gxYwZOnTqFb7/9FsuWLcOsWbPk+gpG+XiKg5sHlybKVBMiIiLXIWu/SXZ2NsaPH4/MzEwEBASgY8eO2Lx5MwYPHgwAyMzMRFpamqZ8dHQ0Nm3ahBdffBGLFi1CeHg4FixY4HDDwKv46LTccJ0pIiIi21MIVRm5LiI/Px8BAQHIy8uDv7+/Ta91q0yFtm+JR0ld/HCkTa9JRETkjCx5fjtczo0z8fbk7SUiIrI3Pn1tyNDwdCIiIrIdBjdERETkVBjcEBERkVNhcGNjf83sJ3cViIiIXAqDGxtr3qi+3FUgIiJyKQxuiIiIyKkwuLGDwe2qVzK/VaaSsSZERETOj8GNHdzfpYnm/fojGTLWhIiIyPkxuLEDN63pbiq4BAMREZFNMbixA3et6IbT+hEREdkWgxs7cONMxURERHbD4MYO3NwY3BAREdkLgxs70I5t2IhDRERkWwxu7MCdEQ0REZHdMLixA+3VwbefzpGxJkRERM6PwY0deLpXBzfbTjG4ISIisiUGN3ag9HCXuwpEREQug8GNHXh5iG9zxs1bMtWEiIjI+TG4sQOlTnBz9xd7ZKoJERGR82NwYwfB/krR52tFZTLVhIiIyPkxuLEDXy8PuatARETkMhjc2MmgtsFyV4GIiMglMLixk3s6NZa7CkRERC6BwY2daM91Q0RERLbD4MZOuDI4ERGRfTC4ISIiIqfC4IaIiIicCoMbOxHkrgAREZGLYHBDREREToXBjZ0wnZiIiMg+GNzYSfvGAXJXgYiIyCUwuLGT8EAfuatARETkEhjcEBERkVNhcENEREROhcGNHa16ujsAIDzAW+aaEBEROS8GN3bk7+MJAFBz0hsiIiKbkTW4mTt3Lu688074+fkhODgYY8aMwZkzZ4wes2PHDigUCr3X6dOn7VTrmqtaXkotMLohIiKyFVmDm507d2Ly5MnYt28ftm7dioqKCgwZMgRFRUUmjz1z5gwyMzM1r5YtW9qhxrVTtXgmW26IiIhsx0POi2/evFn0efny5QgODsahQ4fQt29fo8cGBwcjMDDQhrWzvqrgRmDLDRERkc04VM5NXl4eACAoKMhk2c6dOyMsLAwDBw7E9u3bDZYrLS1Ffn6+6CUXN3ZLERER2ZzDBDeCIGDGjBno3bs32rdvb7BcWFgYvvrqKyQkJGDt2rVo3bo1Bg4ciF27dkmWnzt3LgICAjSviIgIW30FkxRVLTey1YCIiMj5KQQH6SOZPHkyNm7ciD179qBJkyYWHTt69GgoFAps2LBBb19paSlKS0s1n/Pz8xEREYG8vDz4+/vXut6WuHC1EAM+2Ql/bw8ce2eoXa9NRERUl+Xn5yMgIMCs57dDtNxMnToVGzZswPbt2y0ObACgR48eOHfunOQ+pVIJf39/0Usu1Tk3slWBiIjI6cmaUCwIAqZOnYp169Zhx44diI6OrtF5kpOTERYWZuXaWR+HghMREdmerMHN5MmTsWrVKvz666/w8/NDVlYWACAgIAA+PpULTc6ePRsZGRlYsWIFAGD+/PmIiopCTEwMysrKsHLlSiQkJCAhIUG272EuDgUnIiKyPVmDmyVLlgAA+vfvL9q+fPlyPP744wCAzMxMpKWlafaVlZVh1qxZyMjIgI+PD2JiYrBx40aMGDHCXtWuMbbcEBER2Z7DJBTbiyUJSdZ25eYt9Pzwb3h5uOHse8Ptem0iIqK6rM4lFLuKqm6psgo1Hly6F2r2TxEREVkdgxs7qprEDwAOXryBM9kF8lWGiIjISTG4saOqSfyquFaHIBERkX0wuLEjN4XpMkRERFQ7DG7sqMJAjs07G05g9tpjdq4NERGRc2JwY0eN6iv1tt0qU+G7vRfx04F0ZOeXyFArIiIi58Lgxo7c3BSIDPIVbatQqzXvOf8NERFR7TG4sTN3ncQb7Z4qBZiUQ0REVFuyzlDsigpKKjTvxy/bj2tFZZrPTDgmIiKqPbbc2FluYanmvXZgQ0RERNbB4MaBcMJiIiKi2mNw40CYUExERFR7DG7srHmjegb3qdh0Q0REVGsMbuysZ/OGBvex4YaIiKj2GNzYmbERUeyWIiIiqj0GN3amu3imNgY3REREtcfgxs7cjAY3dqwIERGRk2JwY2dGYhsIbLkhIiKqNQY3dmZsEmIVgxsiIqJaY3BjZ+UqtcF953MK7VgTIiIi58Tgxs46RQYa3Dftp2SUlKvsVxkiIiInxODGzkZ3DDe4Ty0Abd7cjNlrj9mxRkRERM6FwY2debi7ITzA22iZnw6k26k2REREzofBjQyYNkxERGQ7DG5kwEFRREREtsPgRgYLHuksdxWIiIicFoMbGXSLDsL594fjuf7N5a4KERGR02FwIxMPdzeoud4CERGR1TG4kVEFgxsiIiKrY3Ajo+aN6tv1ehk3b+H1dSmcCZmIiJyaxcHN5s2bsWfPHs3nRYsWoVOnTnj00Udx48YNq1bO2T3UtQlmDG4FH093u1xv0g+H8OP+NNy76B+7XI+IiEgOFgc3L730EvLz8wEAKSkpmDlzJkaMGIELFy5gxowZVq+gM/Nwd8O0gS3RWWJJhn/O5+L3Y1eser2UjDwAQEFphVXPS0RE5Eg8LD0gNTUV7dq1AwAkJCRg1KhR+OCDD3D48GGMGDHC6hV0BSqJ3Jtx3+wHAHRsHIjIBr72rhIREVGdZXHLjZeXF4qLiwEA27Ztw5AhQwAAQUFBmhYdsoyxtOKcghK71YOIiMgZWNxy07t3b8yYMQO9evXCgQMHsGbNGgDA2bNn0aRJE6tX0BUMaBOMA6nX5a4GERGRU7C45WbhwoXw8PDAL7/8giVLlqBx48YAgD/++APDhg2zegVdwTN9msldBSIiIqdhcXATGRmJ33//HUePHsVTTz2l2f7ZZ59hwYIFFp1r7ty5uPPOO+Hn54fg4GCMGTMGZ86cMXnczp07ERcXB29vbzRr1gxLly619Gs4FHc3hdxVICIichoWBzeHDx9GSkqK5vOvv/6KMWPG4LXXXkNZWZlF59q5cycmT56Mffv2YevWraioqMCQIUNQVFRk8JjU1FSMGDECffr0QXJyMl577TVMmzYNCQkJln4VIiIickIWBzfPPvsszp49CwC4cOECHn74Yfj6+uLnn3/Gyy+/bNG5Nm/ejMcffxwxMTGIjY3F8uXLkZaWhkOHDhk8ZunSpYiMjMT8+fPRtm1bPP3003jyyScxb948S7+KS/hfUjo+2HQKgs5S5EfSb8pTISIiIhuzOLg5e/YsOnXqBAD4+eef0bdvX6xatQrfffddrVtP8vIq52EJCgoyWCYxMVEzQqvK0KFDkZSUhPLycr3ypaWlyM/PF71cycu/HMNXuy5g3wVxwvLjyw/IVCMiIiLbsji4EQQBarUaQOVQ8Kq5bSIiIpCbm1vjigiCgBkzZqB3795o3769wXJZWVkICQkRbQsJCUFFRYXk9efOnYuAgADNKyIiosZ1lMv1ojKczymo1Tnybom7DAtLOJEfERE5J4uDm65du+K9997DDz/8gJ07d2LkyJEAKnNhdIMOS0yZMgXHjh3DTz/9ZLKsQiFOwK3qctHdDgCzZ89GXl6e5pWenl7jOtrS9ln9MbardODV5T9bMejTXbiYazgXyRSdXimjc+sQERHVZRYHN/Pnz8fhw4cxZcoUvP7662jRogUA4JdffkHPnj1rVImpU6diw4YN2L59u8m5ckJDQ5GVlSXalpOTAw8PDzRo0ECvvFKphL+/v+jliKIb1sN9XRobLfPV7gtWu55uDg4REZGzsHgSv44dO4pGS1X5+OOP4e5u2QKQgiBg6tSpWLduHXbs2IHo6GiTx8THx+O3334TbduyZQu6du0KT09Pi67vaCoklmHQboxatT8NH9zbwSrXYmhDRETOyuLgpsqhQ4dw6tQpKBQKtG3bFl26dLH4HJMnT8aqVavw66+/ws/PT9MiExAQAB8fHwCV3UoZGRlYsWIFAGDSpElYuHAhZsyYgWeeeQaJiYlYtmyZWd1Zji6u6R1625b/c9Eq5y6pUIk+s+GGiIiclcXBTU5ODsaOHYudO3ciMDAQgiAgLy8Pd911F1avXo1GjRqZfa4lS5YAAPr37y/avnz5cjz++OMAgMzMTKSlpWn2RUdHY9OmTXjxxRexaNEihIeHY8GCBbj//vst/SoOx9tTv+Xr92OZVjn3Z1vPGd1fWqGCWg34eFnW+kZERORoLA5upk6dioKCApw4cQJt27YFAJw8eRITJ07EtGnTLGpBMSfv47vvvtPb1q9fPxw+fNjs69Qlb4xsi/c2nrL6edOuFxvcJwgCOr6zBaUVaswf2wljOhvP/SEiInJkFicUb968GUuWLNEENgDQrl07LFq0CH/88YdVK+eKRnYMs/s1BQEoragc3j99zRG7X5+IiMiaLA5u1Gq1ZOKup6enZv4bqjl3ieHsUq4VllptxBPTb4iIyJlYHNwMGDAAL7zwAq5cuaLZlpGRgRdffBEDBw60auVckdRcPbr2nMtF3HvbMOvnY1a5JoeFExGRM7E4uFm4cCEKCgoQFRWF5s2bo0WLFoiOjkZBQQG++OILW9TRpZhaILxCpcYXf1cmByccvmyVa+qOQH/5l6Moq2ArHBER1U0WJxRHRETg8OHD2Lp1K06fPg1BENCuXTsMGjTIFvUjHeuPXIGft3Xn8xF0Oqb+l3QZHZoEYnyPplCpBSQcvow7o4IQ3bCeVa9LRERkCzWe52bw4MEYPHiwNetCAFQmuohm/XwU91ppNFOFSo1fj1xBbESg3r6rBaUAgJ8OpOGN9ccBABc/HGmV6xIREdmSWcHNggULzD7htGnTalwZAuorTf+TWCtHZkXiJcz5/aTkvgV/nUO/Vg1x+NINzba84nIE+NbtWaCJiMj5mRXcfPbZZ2adTKFQMLipJV8vD4T4K5GdX2qwzPojVwzus8Tef68Z3f/IV/tFrUSxc7YgcfYAhAX4WOX6REREtmBWcJOammrrepCWuKZ3YFNKlumCAL7dk4one5tek0uKqYFZZSo1PD3Ehf4+nYNx3ZvW6HpERET2YPFoKXIsc34/iZNX8iX3meq+Sk67YXQ/AHi4iX9EOGqciIgcHYMbJ5BbWNmFVVBSjr9OZWuGcRvr2qo8rszkub08+CNCRER1S41HS5HjqGpMeWZFEvZduI5n+kRjYs8o9P5oe63P7eluvO9KEAQUl6lQz4xEaCIiInvgn+UOSAHzlmCoUtX9tO/CdQCV89RYI7AxVJf068V4+vskHLx4Ha8kHEPM23/ieEaeVa5HRERUW/xz2wH1aBaEjSmZZpcXAHyz+4LtKqRj6k/JOJJ+E9tOZWu2LdnxL9qE+mH7mRz8+HQP+Hi5260+RERE2moU3Ny8eRMHDhxATk6O3mKZEyZMsErFXNmj3ZvizV9PmH+AALy38ZTmo5lrb9aIAOBUpn4Cs7ubAp9sPQsA+F9SOib2jEJ+STncFQpRl9Wus1dx8OJ1TB/UCu6m1pogIiKqAYuDm99++w3jxo1DUVER/Pz8RAs9KhQKBjdWYOlDX3f5BFuHDKUS6055aNX5xJU8PL78AHacuQoAuPDBCLjd3j/h2wMAgKgG9XB/XBMb15SIiFyRxcHNzJkz8eSTT+KDDz6Ar6+vLepEFtIdnq27EGatzg3zTuamFdz8L0m8oGeZSg1vN3E3VfqN4tpXjoiISILFCcUZGRmYNm0aAxs7+3J8nMF9usFN3q1ym9XjzdvrTOnysLC1yZoBGBERkTaLg5uhQ4ciKSnJFnUhI/yNrAReUGq7YMZcxrrS1BIz/1lrfSwiIiJdFndLjRw5Ei+99BJOnjyJDh06wNNT/NC9++67rVY5qhbgYzi4efc36cUv7clYy41UHCMV8BAREVmDxcHNM888AwCYM2eO3j6FQgGVSlX7WpHI6NhwtA3zM7j/ZrHtWm4ybtwyq5y7m+FGQJVEIFPVLVWuUsPTndMtERGR9Vj8VFGr1QZfDGysz8fTHV880lk0Ks2ezF2B3Fiej6A/uAqCABxIvY6Wr/+Br3fZb44eIiJyfvyT2cHJFNNYLOHwZYP7pFpuBEHAQ18mAgDe33RKbz8REVFNmdUttWDBAvzf//0fvL29sWDBAqNlp02bZpWKubrWIX44k12AoTGhmm0LH+2MKauSZaxVzajUAhb8dQ6tQ6u71phzQ0REtqIQzBi2Eh0djaSkJDRo0ADR0dGGT6ZQ4MIFx+5iyM/PR0BAAPLy8uDv7y93dQy6WlCKLSezcE+nxqivNcNv1KsbZaxVzXw2NhYvrjkq2vZU72gs25Oq+Xzxw5Gi/Sq1AHc3BdKvF+Pgxeu4p1NjzmhMROTCLHl+m9Vyk5qaKvmebKeRnxLjujfV2z6pX3Ms3fmvDDWquay8Ur1tui03giBg34XraBvmh0+2nMX65Axsm9kP/eftgEotoKCkAhN7RtmpxkREVJcx56aOGRoTIncVrEK3vfDLXRfwyNf7MPzz3fhh3yUUlFbg2z2pUN0eVrXr7FUZaklERHVRjRbOvHz5MjZs2IC0tDSUlZWJ9n366adWqRhJc5Zh07otNx/+cRoAkJlXIlleaj0rIiIiKRYHN3/99RfuvvtuREdH48yZM2jfvj0uXrwIQRDQpUsXW9SRtMSEO26ekCXMSSjWLlGmE9xk5t3C6awC9G/VqNbD5JMuXseR9Jt4qne0bEPuiYjIeixuBpg9ezZmzpyJ48ePw9vbGwkJCUhPT0e/fv3w4IMP2qKOpKUuPnylqmzpYKnSCvEcSvFz/8YTyw/izxPZBo/JLSzFP+dzTS718MDSRLy38ZTRcxERUd1hcXBz6tQpTJw4EQDg4eGBW7duoX79+pgzZw4++ugjq1eQ6r71yRl628xZOFM7KDHULbX331yDx9/18Q6M+2Y//jieZfpiAFJzi8wqV1Kuwo2iMtMFiYhIFhYHN/Xq1UNpaeXol/DwcPz7b/XIndxcww8acl2nswr0tpmzcKZ2kTKVdHBjrHuroLQCAPD36RyT17LEne9tQ+f/bGWAQ0TkoCzOuenRowf++ecftGvXDiNHjsTMmTORkpKCtWvXokePHraoIzkhSyfxKy2XDm7MOY25HXnm9vhVBU1HLt/EXa2DzTw7ERHZi8XBzaefforCwkIAwDvvvIPCwkKsWbMGLVq0wGeffWb1CpJzKi4zvQ6ZdtxiqFvKmvMcCwKw+9xVXCssw5jOja14ZiIisieLghuVSoX09HR07NgRAODr64vFixfbpGJk2GM9IrFyX5rc1aiV349lmizz29HqRTt1E4qrWHsVh/HLDgAAOjYJQLNG9Y2WrXup3URErsGinBt3d3cMHToUN2/etFF1yByvDm8rdxXsIqegemZjgy03t6Ob7PwSpF8vtvgaFQZyebLz9WdV1lUXR64REbkCixOKO3ToYLX1o3bt2oXRo0cjPDwcCoUC69evN1p+x44dUCgUeq/Tp09bpT51hae76z1U1QaGVwlCZYDT/YO/0Oe/21F4Ox9Gm24MIggCnv/xEN7+9TgmfHtA+rxW7fAiIiJ7sji4ef/99zFr1iz8/vvvyMzMRH5+vuhliaKiIsTGxmLhwoUWHXfmzBlkZmZqXi1btrTo+LpO6eGOZRO76m0PC/A2eey47pG2qJLNGQo1BAiaJRqAyhYcyXKCgKPpN1FUWoEz2QXYlJKF7xMvYe+/1zRl2BBDROQcLE4oHjZsGADg7rvvFjXLC4IAhUIBlcp0omiV4cOHY/jw4ZZWAcHBwQgMDLT4OGcysK3+GlNjOjfGkh3GF9WMCPK1VZVsSqUWND9j2gRBHPgkp91Ec51cGQUU2JiSiSmrktGxSQDeH9NB8hqpV7XmudE66Z8nsrD6QBrmPRiLBvWVWuclIiJHZHFws337dlvUwyKdO3dGSUkJ2rVrhzfeeAN33XWXwbKlpaWaeXkAWNy6VJeUV6jRoJ4XrhmZf0XpUXfXpkrNLdJL8hUgHlY+6+ejeCCuid6xq/ZXJmAfu5xnsMtpTVK66LxVnv3hEABg7h+nMe/B2BrWnoiI7MXi4CY6OhoRERESf0ELSE9PN3CUdYSFheGrr75CXFwcSktL8cMPP2DgwIHYsWMH+vbtK3nM3Llz8e6779q0Xo6iQi0gvnkDoyORGvkp8VTvaLi7KbD/wjUcvZxnxxrWzvxt5xAZ5Itn+zXTbFMLgskRUwoFUFBSnYtjzggrqTK5heIkY3ZjERE5phoFN5mZmQgOFk9edv36dURHR1vULWWp1q1bo3Xr1prP8fHxSE9Px7x58wwGN7Nnz8aMGTM0n/Pz8xEREWGzOtrTuud7YsFf57D9zFUAQLlKLco/keKmUODNUe0AAEWlFYh5+0/JcsPbh5q9bIG9bLg9NFzUMiWYNyFgcVl1cLNy3yWr142IiByHxX0UUnkPAFBYWAhvb9MJrdbWo0cPnDt3zuB+pVIJf39/0ctZdI68A8uf6Ibn+jdHoK8nnuvfHBUmgxvt94abHu5q47gz76Zk3NS8r+yWMn2MdpGfD102WT71WhE+2nwa1wpNDwknIiLHYnbLTVXrh0KhwJtvvglf3+rEVJVKhf3796NTp05Wr6ApycnJCAsLs/t1Hckrw9pg1pDWcHdToHNkILaeNLy6tXZgaqxbpa4MNxcEQa/l5lx2AVqG+Gk+rz5oeXfpm+uPA4Begrb2mlgKphQTETkks4Ob5ORkAJW/3FNSUuDl5aXZ5+XlhdjYWMyaNcuiixcWFuL8+fOaz6mpqThy5AiCgoIQGRmJ2bNnIyMjAytWrAAAzJ8/H1FRUYiJiUFZWRlWrlyJhIQEJCQkWHRdZ+R+u0nm6d7N8N/NZwyW026tcXcTP5zv8PXEjeJyvXKOTC0AFSpxcDP4s13o1aKB1a9VOaeOsf3SrZpERGRfZgc3VaOknnjiCXz++edW6d5JSkoSjXSqah2aOHEivvvuO2RmZiItrXqZgbKyMsyaNQsZGRnw8fFBTEwMNm7ciBEjRtS6Ls7CS2s0VCM/Ja4WiLtVjHVLff5wZ4OT2jkS7RYTAcCKxIt6Zf45f01vmzUYim2+33sRS3b8i5VPd0eLYOPLNhARkW1ZnFC8fPlyq128f//+omZ+Xd99953o88svv4yXX37Zatd3Vm6KyhaNDo0D8PfpHNE+fx9PUTlt3aKD7FE9qxIEAVtOGO6Gs+q1IE5e1o4N395wAkBld9bc+zpgXXIGnuwVjQBfTxARkX1ZHNyQ49s8vS9WH0jH83c1R9f3tmm2P9ErCl2b3qH5rN2F4uGmgLenO94a1Q63ylVoUE+JuuD3Y5l2bSkxNTCrQq3GiAW7UVymwsVrRfj84c72qRgREWnU3RndyKBWIX54a3Q7NKwvDlDeHh1jMCdk+6z+AIAne0dj8l0t9PbH3Q6K/LwdLx4+n1Not2uZs+ZUcVnldAhJF2+YdU6VWsDxjDyTw/iJiMg8DG5c3A9PdcOiR7voLcugGwMtGdcFU+5qgTX/F2/H2jmem7cTrgHTrTgeZo44m/PbCYz6Yg/mbjplVvn068X4ate/KCgpN12YiMgFMbhxcX1aNsLIjqaH0gf7e2PW0NaIalgdBP0+tTf6tmpky+pJknNA0ugv9mjeT1+TDJVawDMrkjTbtAMedzMr+n1i5aSC3+xJNa8OC/fgg02n8e5vJ80qT0TkahjcuAjd5OGa8nKv/pHx9nTDiie7ockdPtY5uYMTBAE5WqPPcgvL8O/VQoPzCrlZ66brqGo9SvzXNiPCiIjqOgY3Tu6Hp7qhWcN6WG1hd5KhLhcPreCmcWBlK87a53uiRzPxSKuOTQIsq2gdpRu/aN82DxsFN0REZByDGyfXp2Uj/D2rv1WHeZ94dyiOvDUYPl7uAIBgP2/c27mxqMzQmFCrXc+RGcsBPp1VoFmNXNfuc1dx3+J/cC67wEY1M+1UZj5m/XwUl28Uy1YHIiJbYHBDFqun9ECgr5dom+5DfmLPKJtd/5gDrWSuu/TDmSxxsPLauhTJ48YvO4DDaTcxaeWhGl+7trlHo77Yg18OXcZzKw/X7kRERA6GwQ1JsvTBqfuQr6/0wMZpvUXbjrw1GEPahWg+D2obgrpOrRZ/LiytkC5owHXtFc4tVNvgpmrouW5ARkRU1zG4IUlKD8t+NLRbbubcEwMAiAkPwBO9ojTbA329ENWwnuZzXVuGafe5XL1tukGdpW4U13w4t60W7vwh8SL2ntf/rkREdYXjzchGDqFLZPVMxkH1vIyUrKS9jMaE+CjNe90HsPYil86Qb1vL2Mbh7L9wDW/+WrmUxMUPR8pcGyKimmHLDUlyc1Ng1dPd0aFxAFY82c1keUMPed0ARqXVj6OAAt2i6t56VtpUZkY3KZfzsP1MjumCWsoq1Nh4LBPXCquHn6ddq07+tUXLV9p1JhcTUd3H4IYM6tmiIX6b2hvtG5se1m2oe0b3AazSWXhyxVPdMDSm7ubejFn0j8kyFSo1Ri/cgyeWH0R2fonZ5160/TwmrzqMB5cmarY9tmy/5r2lsU1ecTnUUsO7tE7kZA1RROSiGNyQVRhqwNBdy+qZPs209gHenu6ICXfuOXEy86oDmiILEo43pWQCAC7kFmm2abesVN3b/JJy/N+KJGw8lmnwXKez8hE7ZwsmLj9g/KKMbojICTC4IatQekr/KAX4eIo+N22glVB8u8lgXPdIvXLOpKYLYprb7bRo+3lsOZmNyasMD+n+cV/lfDtSSdFERM6GwQ1Zxf1dmqB7dBBeGtpatP2JXlHo16oRPri3g/5Btx/eDeorcfjNwQbP3adlQ3z+cCcr1ta+tLvsVh9MFyVf10ZV7HO90PRwcnOTt81Z9ZyIyNFxtBRZhbenO9Y8q7/Eg6+XB743kJCs/bx1N/D0HdkhDG+MaovLN25Zo5qy0G64+WrXBURptV4ZY3Ko9+3d5rTw6HYP6hIEAZ9tPYvDaTfNqhsRkSNjcEN21zjQBxk3b2FEB9OrkS8a1wUAkJNfaqKk49JtqTE0a7EuU0GLNQdL7btwHQv+Pm/FMxIRyYfBDdndphf64HxOgWguHV3DYkLxTN9ozWcvCycVdCQ1TLkRUakFg61b5tAOlPZfuIbuzRqI9tdmpmQiIkdTd58YVGcF+HgirmmQXlfJsold0SbUDxun9cbS8XGIa1o9B46lMyY7krNWWBxzy4ks3CwWByBV989Q99VvR6/gwaV7kZVXIioz9qt9emWdYUJFIqIqbLkhhzGwbQgGGlhvylotN24K67SkWGLqT8kWlb9y8xZeXZuC01prPiVduoEG9ZWiclXxiHaMmHHzFhoH+oiuO+f3EwgL8DF6TVM5OUREdUnd/XOYXIqHm+U/qi2C6+tt++8Dsdaojk29uf44dp29Ktq2bE8qHvoyUbRNKh7p9eHfmvlxquTdKjeZn2MotimrUONw2g1UqNTSBYiIHBCDG6oTahDbiB7o93VpjMd6RDp899aNojL8ddqyZRp0A5PnfxTPd5OVV4Jv9qQaPh6Am4Ho5tW1x3Df4r34+M8zFtWJiEhOjv2bnui2mkzyp71sxKcPdcJ7Yzo4/CwuDyzda3bZs9mFWHMwDVLjprRHaP17tUhv/+y14hFb7gZ+E6w9nAEA+HLXBbPrRUQkNwY3VCcoPdyx86X+Fh3z9uh2eKp3NDZO663ZZq0J9GxFKhAx5pUE6WHl45cZX2bhpwNpmvelFWq8uOaoXplfDl3W27b331yrJEgTEdkSE4qpzmhq5uR3VQJ9vfDmqHaibdqxTZtQP7QL99e0TtRVUj1Ke85btsxC3q1yvW1v/Xpc9PlibhEe/bpy4c6LH4606PxERPbElhtyKT2bV87v0rC+Epun90Vc0+q5dra82BdLbk8aWJes2p9mulAN6MZMZ9hiQ0R1BIMbqlNeGdbG4L7He0Zp3n/yoPSoqGB/bxx6YxD2vHIXAHFLTqsQPww3MmuydiDkCorKVKLPz/5wSPP+wtVCPLQ0ETt1RnURETkCdktRnfJc/+Z4rEckPt92Duk3ivHq8LbYciILft6eGHtnBKYNbIkAH0+js/lqzxdjSQbOoke7oMfcv2pRe+cxaeUhnM0uxIFvD2i6qI5n5KGe0gPRDS3rPiQisjYGN1Tn+Hl74g2tXJpn+zXXvA+q52XRucxJMP57Zj8oPd05i6+WDJ2FTK8WlGLUF3sAMB+HiOTH4IZcWp+WjQAYH2rerFHlZIA5BSV2qVNdoNtllX6jWKaaEBHpY84NubTohvWw++W7sPfVASbLGproTsqANsG1qVado31n3v71OM7nFMpWFyIiBjfk8iKCfFFPaboR01RwExsRaKUa1T3aa1N9n3gJYxb9U6vzvfLLMTy+/ADUdl4ITK0WzL7mhqNX8GrCMZRzaQoih8NuKSIzGQptVj3dHUH1vXAk7SaOpt8E4PiTBVqb7r0pLK1AhUoND0NTH5uwJikdAHAyM1800zRQGYAs3H4enSIC0bdVoxqdX4ogCBj5xR6o1GpsfqEv3EwkWU27vTBp58hAjL0z0mr1IKLaY8sNkQ7tGY21GWq58ffxRJtQfzwQ10SzzbVCG+mJBFfuu6R5r1YLOJtdALVawPGMPAz8ZAe2nMiSPJepwPCP41n4dOtZTPjW+CzMlsq/VYFTmfk4m12Iq4WlZh93rajMqvUgotpjcEOkIyY8AJFBvvo7tB7g79/bXvO+ath5TVspnEFOvn4wkHy7FQsA5m05gyGf7cIHm05h8qrD+PdqEf5Pa94cbaYavWyWvKz17+tiDW9ETsd1fxsTGSFItL1o91J4agUyHhLdF670cPz3aiGeXpGkt137Hize8S8A4Js9qbilNdLqQOp1vePUWgdakMNda9rXkvr3N3icVlS08Vgmpqw6jOKyCmtWjYgsJGtws2vXLowePRrh4eFQKBRYv369yWN27tyJuLg4eHt7o1mzZli6dKntK0oEcbeUp3v1e1O5GYYMauscI6oe/Xqf5HZD4YGXR/WvnYe+TMTr61KQnHYDAHDpWhGOX8m3dhUN+vVIBqb9lIyScpUob8iSPGbtoGjyqsP4/Vgmvt6VarU6EpHlZA1uioqKEBsbi4ULF5pVPjU1FSNGjECfPn2QnJyM1157DdOmTUNCQoKNa0qupp6Xfq699kPMw81Ey43O5//cEyP6vHhcFyx9LK5WdXQU2RJdUoDh3Bnt4AYAftyfhnsX7wUA9Pt4h2ik1ZH0m0i/brs5dF5YfQQbjl4R5QcBtU8Iv15kfs4OEVmfrKOlhg8fjuHDh5tdfunSpYiMjMT8+fMBAG3btkVSUhLmzZuH+++/30a1JFc0/+FOmLIqGTMGt9JsM9RyI5Vr01QnZ8dfZ5LAEUbWsHIWhuIDLwO5SVJDql9fV7kyufasx5bEHWq1gENpN9AuzN/ocP/cwjJRQCp1DbVawCNf70PD+kos0lpgVardTmHP/jQi0lOncm4SExMxZMgQ0bahQ4ciKSkJ5eXlkseUlpYiPz9f9CIypU2oP7bN6GdWEFJfq5Vnzf/1wCPdIjBraGtRGVfKwaliKG9Ft+WmSlGp8TyVknIVVu67hMsWJBT/eCANDy5NxKPf7Ddabn1yhujfSOrfK+PmLexPvY6NKZnIL5H+fUNEjqFOBTdZWVkICQkRbQsJCUFFRQVyc3Mlj5k7dy4CAgI0r4iICHtUlZycdk6Gr9Jd8757swaYe19HveUcvD3r1H81q9h9LhcqieQVQ60aBSWGg5vC0gp88fc5vLH+OH7cn2Z2HX65PV/OUa2RWwCw6+xVnMkq0HzOyi8R9SV+tPk0SivES0x4aLXWZeUZX4qj6it+uuUM7pq3AzeLOVycyJ7q3G9c3V+MVX3jhn5hzp49G3l5eZpXenq6zetIzsnb0x3P9W+OJ3tFI9ivemVxTwPdLIPaVgfid0YF2bx+jqagpALL/9FPrDWUf33CSCJx+7f/xJ5z0n/AAMC7v53AJ1vO6G2X6jI8m12ACd8ewND5u0TbtVuaNqZk4utdF0T7teM07aBN6ldPVRfmgr/PIzW3CN/uYYIxkT3VqRmKQ0NDkZUlnvgrJycHHh4eaNCggeQxSqUSSqVSch+RpV4Z1gZA5cOtf+tGaH57UU0pi8d1wbmcArQL88et8upWgM6RgbaupsNYezgDT/dpJtpmKBtl26nsGl0j/Xoxlv9zEQAwfVArzbxDAETvq5zNLtDbBuh3RZ3TWR9Le1kG7bIKiW+ku6VcLeDElTzsOHMVT/eJhtLDXe8YIrKeOhXcxMfH47fffhNt27JlC7p27QpPT8OrOhNZm7ubAt890c1oGS8PN8SEVy4d4OvlgeWP34mcghLc36V6JuPF47rg+R8P27SuclILgl4LiKGZnitMrNFkKG2pTOs4tSDA/XZo8UdKpuQ8OikZeWadv0KnS007oFGbSKLSnR5AEICRC/ZU7lMo8Fz/5kaPJ6LakTW4KSwsxPnz5zWfU1NTceTIEQQFBSEyMhKzZ89GRkYGVqxYAQCYNGkSFi5ciBkzZuCZZ55BYmIili1bhp9++kmur0BktrskVgqPblhPhprYj1oQ8P6mU6JthoKbcpXxgMGcpGztoOM5naBREAT8dSoHX+68oHuYZr/oXDrBjfa5RS03El9Hd5N2l9fJTA5qILI1WXNukpKS0LlzZ3Tu3BkAMGPGDHTu3BlvvfUWACAzMxNpadXJg9HR0di0aRN27NiBTp064T//+Q8WLFjAYeBUZ/l4Wt498VTvaL1t1lxA0pqkJsMzNEra1Ora5swarL59Cu1k4SorEi9JzqRcfX4x3WRo7eDGVMuNXg6gqBuLiGxN1pab/v37G50s67vvvtPb1q9fPxw+7LzN+ORaohrWwzN9ovH17lS8MLAlPv/rnMlj+rVqhGU6CaodGvtj19mrtqpmjUkFLIaCG91uIF1qidhn7qZTohymv0/nYGTHMBy4qN8dtSLxotHzv/LLMfH1dH433bxVPfz78O0ZlQHpwQxGYhsisoM6N1qKyNm8PrIdLn44EhPim4q2G5oPRimx3dCILblduqY/J42hbqlCI0PBAenWki93XcCkldV/7Exedfu9RFlT3Vp/nc4Rfb5845ZoPpv7bs+iDBjO26mim8esNjG6ioisyzF/IxK5IO0WgPfGtMeRtwbj8JuD9cqFBfjobXOG0TdSrS3aajsRoqWHn84qQMd3tpisi+QMxex8IpIVgxsiB6H9137P5g3g6+WBoHpeeuUiG/jqbdNeDsLRmcqtMcRUnkuV3h/9jVMSOTfWZGqem9p0Sy3/JxWPfr2PK4sT1QKDGyIHod1yY2ptosXjuoiSkQ11YRnzSDd5Zus+ePGG6UISzA1uLt+4hVUSsxjXZjHMf86LJxA0VZebxeLlGcytOwC8+9tJ7P33Gn5IvGS6MBFJYnBD5CC04xlDs/hWGdEhDMffHar53KtFQ7QKqZxQcMbgVnj4TtOBy+wRbfH9k93wUNcmJss6gtp2S12UyP8xx42iMozTWZtKuypS/1Q/7LtkZNI/82hP/EhElqlTk/gROTPtRFvtnI3GgT7IuHlLr7y7mwJbX+yL3MIyNG9UH3+80BcKVE4g94HO3DJA5SirnbdHVE0f1BL+3p7o16oROjUJxP+SLlv/C1mZJa0f1rRo+3m9bTu0ko9XH0zHobSb+PShWFGZcqnhXRZg3g5RzbHlhshBaD/KtFtxvn/S8EzILUP8EN+8cukRdzeFZmbcJnfoJx1rj6iqr6z+uybAt27M7l3TlpfaOpWlP+leUVl1q8rprAL8dvQK/pckXrdOO7bR7hIz1eVYXc7CihKRBoMbIgdh6GHWIrg+IoL0gxVjHukWicd6RIq2vTGyrcHy6yf3suj8ruSf89fMKvfZ1rOizxVa0U1N2pxMdU0SkWHsliJyEMam9H9hYCvM+vmo2efydHfDe2M64KUhbZB4IRd3tQmGl5G5cDpFBFpYW9KVW1gm+pyaW6R5X2BiDh8p5rbwEJE+BjdEDkI750Z39FPjQMtabqoE+HpiWPswve0ypa+4lLsX/qN5vy45Q/PeUMhy4WohTlyp7gJjbENUcwxuiByEj5c7pg5ogbIKNYL9vEX7ejQLwqwhrdC8UX2rXMucdZqqfDU+Dv/3wyGrXJcMG/DJTtFnQzM5E5FpDG6IHMjMIa0ltysUCkwZ0NJu9Xh5WGt8u+cifniqG9qG+dvtui7BzJiFoQ1RzTGhmMgFSXVLDWobAgBoWF+J5/u3wMHXBzKwkRFbbohqjsENkQuS6pSa92BHPN+/OdY82wOA4YTWkR3FOTwc1WMZc+evYWxDVHMMbohckFTLTaCvF14e1sZoXo+nuwKLHu2CWUNaabadeHcYDr85GH1bNcLbo9uhTaifLarsNHIKSnA+x/TaVxwtRVRzDG6IXJAlCcXaohrUAwBor33p4+WOoHpeWPFkNzzRK9oa1XNqu8/lYtCnu5CTX6LZllNQoleOoQ1RzTG4IXJBlg4F/31qb4zoEIqvJ3QFYHwpBLY4mOf07ZXLi8sq0O39v/T28zYS1RyDGyIXVNUCY672jQOweFwcohpWHmcsNnphYAuT52sT6odFj3axqA4AENXA1+JjHNXxK3lI/PcarkisGwYwoZioNhjcELmQ1f/XAy8Pa43h7UNrdR7BSMvNsPZh2Dd7oNHjn+nTTC8x2RyBvl4WH+Oo/rv5DB75eh+y80vlrgqR02FwQ+RCejRrgOf7t9AssFlTXZreYXR/aIC35PapA1rgf8/G474ujWt03U8eisXQmJAaHeuoDLXcvL3hBErKVZL7iMg4BjdEZLH+rRrh6wldseuluyw6rmF9JbpFB9U4L6d5o/pY+lgcOkcG1uh4R/TSL8cM7vvnfK4da0LkPBjcEJHFFAoFBrcLQaSFOTAN6yvNKrdsYlej11br9IrVdvh5i2DrLGthbfO3nZO7CkR1EoMbIrKph7o2wfv3tscDcU0wzMxcn/6tg43ur9Aei47aLwS69LG42p3ARlIy8jDr56NGc5yISB+DGyKyuXHdm2Leg7Fw18n1WT+5F14Y2BKBvp6i7e5uCsQ3a2DwfCqdppsKtdpASfN4OPA0y78cuowj6TflrgZRncLghohsqp7S8Pq8nSIC8eLgVpLBhZuR307lOi035aratWwoPR37V2F+SYXm/Yd/nMbCv9ldRWSMY/+PJqI6670x7dE5MhDTzFjNXPvhvfQx6flvOkUEat7rttwUlVagNsICfGp1vK1VhX6ZebewdOe/mLflrF6AR0TVDP9JRURUC4/1aIrHejQ1q2xZRfWDeki7yrwc3QUmXxhUHSTpttQUlNQuuDFXdMN6SM0tssu1tFUNListr75PxmaJBirzkjzc+fcruSb+5BORQ6mag0d7tPgj3SLQr2UjzecynVYL3c+1NaSd/lw6Desr4ectz9+DuYWleHHNERxOu6HZZizNaOHf59D6zc04ylwdclEMbohIdn6383K0h2Q/3acZAGBQ22DMva+jaOJB7ZYeAHiuf3OD524Voj/M+7EekUbr4+Uh/tX46vA2WD+5p2yLWc757STWJWdgxv+OarapjLTczNtyFiq1gLc2nLBH9YgcDoMbIpLd6md74O7YcHw78U7Ntn6tGmH/awPx1Xj9OW+0801O/2cYYpsESp53VMcwfDa2k2jbwdcH4T/3tDdan94tGmreL32sCyb1a44md/jKtijojeJyvW0qtYBDl26g2/vbsOHoFcnjHHcMGJFtMeeGiGQXEx6ABY901tse4i+9jIN2y423pzvuatNIspxaEBATHoAezYKw78J1AEAjP9MTCT7UNQL1vT3QOfIONA6sTjbWHsru5eGG5o3q46WhrfDkd0kmz1kbCoX+XD7PrEhCyuU83CpXYdpPybg7NtymdSgsrUBxaQWCDfybEDkSttwQUZ3T5A7x6CalhzsufjgSFz8cKdo+umPlA797tP6cOY/3jAIAPNErSm+fm5sCozqGiwIbQNwSEt2gHv54oQ8GtLH9WldKD/1f1QdSr+NWDdeeyrh5Cw8u3YvNxzPNPqbjO3+i2wd/IbeQC32S42NwQ0R1zjcTu2Jgm2BsmNJLb19LrbydqhmRpXqT3hzVDhum9MIbI9uZfV03rRMZGq20aVofs89nLm9Pd6uc51aZCgv/PoeHlibi4MUbmLTysNnHVo2+T7mcZ5W6ENkSgxsiqnNaBPth2eN3oqNErs23j98JNwUwplO4JkfGTSK6cXdToGOTQL1Zk43RXg3dUHDTLtxf9Fmqu81SNyVybmpi/razmLflLDIMrEQu5XhGHp7+3rbdbqaodRcTIzKBwQ0ROZWIIF+ceHeYKJHYWom1LwysnmvH2ON2TKfq/BcvO801cyT9JraezDZaJrkGQ8PvW7IX205Vn1cw+s2tL6egBF3f34Z3f+PILzIfE4qJyOn4eIm7cSwZ5ORtZCkG7fMam0Pvs7GdsP5I1Qgm+wQDYxb9o7dN93tL3Ybisgr4eLqjqEyF+hJLZegOu7f3Gp7L9qTielEZlv9zEW+PjrHvxanOkr3lZvHixYiOjoa3tzfi4uKwe/dug2V37NgBhUKh9zp9+rQda0xEdU3jO0wvr7Dyqe5oHeKHVc/0MOucxmYI1h4y7ugLev938xlMW30E7d/+E/9eLTRZvur7bD6eiZX7Lll0reS0G0i7VlyTahJZRNaWmzVr1mD69OlYvHgxevXqhS+//BLDhw/HyZMnERlpeJKtM2fOwN+/ul+7USPpYaBERABwd2xjnM0uxJ1Rdxgs07tlQ/z5Yl+T53pzVDt8sOkU/nt/R719Uuk7csY26deLcflGMZrc4WuwzO5zV/Hv1colJX5IvIR37jbeOnK9qAwANMnIvVo0RHTDepJl9124hqsFpRgdG47U3CLcu3gvAOiNaiOyNllbbj799FM89dRTePrpp9G2bVvMnz8fERERWLJkidHjgoODERoaqnm5u1tnJAEROSd3NwVeGdbGKsO2n+odjdP/GYbuzaqHl1fl1egmEwPyttzkFpah90fbNQuNSnXPadfP38fT5DlfTjgman25UVwZ7Hy581/cs3AP8kuqk58f/mofpv6UjPM5BTiVmV+zL+HgLV/kmGQLbsrKynDo0CEMGTJEtH3IkCHYu3ev0WM7d+6MsLAwDBw4ENu3bzdatrS0FPn5+aIXEVFteOokCf86pRceiGuCpY/F6ZXVTcB9qGsTm9ZNStWMzrqLkQLiZRyM5Rtp26Q1P07VSLS5f5zG0ct5+P6fi3rlr9wssaS6RLUmW3CTm5sLlUqFkBDxX1IhISHIysqSPCYsLAxfffUVEhISsHbtWrRu3RoDBw7Erl27DF5n7ty5CAgI0LwiIiKs+j2IiNqG+WPeg7FGu3+qzLmnPebcY9/E2CU7/gUAJF64prdPpTXM2txWJu1yul1xpRX6K3oKFpybyBpkHy2lu1aLIAgG129p3bo1WrdurfkcHx+P9PR0zJs3D337SveVz549GzNmzNB8zs/PZ4BDRHaj/VD38/aAt6c7JsRH4a1f7Te0+fO/zqG4rEJyn0pnDpm0a8VIvJCLezs30VtAtIp2a5SbQiFa68vQyLQaDyHnAllUA7IFNw0bNoS7u7teK01OTo5ea44xPXr0wMqVKw3uVyqVUCpNryVDRGQL2o903e4se/p6d6rkdu2h3h//eQYf/3kGQOVinZP6Sa+2rh2wPfLVPjzcrfoPRqlYRLBhs82+C9cwY80RzLmnPQa1s/1SGFQ3yPY/zcvLC3Fxcdi6dato+9atW9GzZ0+zz5OcnIywsDBrV4+IyCq0H+weWn04Hz+gP9pKDlLdSEBl0PDrkQzJfaezCjTvC0orxIGTRNPN48sPYssJ4xMMGqQVF1Wo9Os67pv9uJJXgqdXyDuLMjkWWUdLzZgxA9988w2+/fZbnDp1Ci+++CLS0tIwadIkAJVdShMmTNCUnz9/PtavX49z587hxIkTmD17NhISEjBlyhS5vgIRkdm0g5sHu+p3jxsbqm4rupP0VSksqcALq49I7vvt6BXJ7QCw40yO5PYNRo4x17pk/WBLt1uNCJA552bs2LG4du0a5syZg8zMTLRv3x6bNm1C06ZNAQCZmZlIS0vTlC8rK8OsWbOQkZEBHx8fxMTEYOPGjRgxYoRcX4GIyGweRrqlGgf64OMHYjFvyxn8fiwTH9zbASpBwJvrj9u0TmUSrSEAkHTpRo3Od+xyHs7nFKKF1gKmuozlVhpz8VpRjepErkf2hOLnn38ezz//vOS+7777TvT55Zdfxssvv2yHWhERWUfjQB+M7RqBNUnpmDmklWSZl4a2xuS7WgAAPn+4M14c3ArNGtbDpTo6m292fonR4EalFuDhXh3c/H06G1/uvICPH4hFZAPDI86kFkAlkiJ7cENE5IxWPd0dF3KL0DUqCF0i78ALg1oiPFB6GYhgv+pBD+5uCjRvVF/zXtvZ94bj+R8PYdsp6a4fR1FP6WE0iVi3J+nJ7yrzZV5OOIrV/xdv8LiatPaQa5J9bSkiImfUs0VDPNajsovdzU1hMLABDM8Bo926MX1QS3h5uOGThzpZs5o2kZ1fohfAaDO0Lte1wjKj55Va3oJICoMbIiKZGXrYu2u1VNwZFQRAnJTsqJ794ZDRlpvzOdILdN4qVwEAUnOL8M3uCygsrcDhtOrcH3cHa7nZ+28uJq86jKw8zsDsaNgtRUQkM0N5JtrdUlXvtVtzHJmxlptRX+zBzpf6o2mDejifUz2s/PKNWzh5JR9jFv+Dsgo13tt4SnScmwMFdr8cuoxZPx8FAOTfKscPT3WXuUakjcENEZFM/vdsPM5mFyBeaxFObdrBTdUEgB5udaPB3dSMxAcv3sDVglI8sDRRtP2Lv88ZHJ6uq1xnpFdNR2HVRFVgA3AUlyNicENEJJNu0UHoFh1kcL/2g7pBPS8A+knGjupqQanJMhtTMvW2FZZKLxMBABUqccD06Nf7RJ+jZ28CAOybPRChAd7mVNMq1ObFYnXWoUvX4e3pjpjwALmrYra68ScAEZELquflrnkfFlj9sI6NCJQs3zrEz9ZVMlvvj7abLCO1HIWx4OazbWexT2vxz4MXpefieWF1shk1tB5bLi8ht9zCUty/JBEjF+yRuyoWYXBDROSgPNzdcPjNwUh+czCUHtWBzq+Te+HEu0PxRK8oUfnJAyrnyrmvS2N7VrNGBEGQTI4+npFn9LjX1qWYPPeZ7AKTZazJkSZJTrmchzfWp+B6kfGRZ+bKzq9Olq5LQRyDGyIiBxZUzwt33O6S0lY5l4x4W7swf5ycMxSfPBhr1ToMsdGClFLBTbnK+AO0sKQC/5zPxbVCw91eN4vLNe+/33sRb6xPsemD2dBoNzmMXrgHK/el4c1fpWe2PnTpBsYs+kc0Cs0Y7YkT69JSFwxuiIjqqPHxTUWf3d0U8PXygEKhwCPdIq1yjdP/GYYnekVb5VzaSirUNRr9lFNQinHf7Efce9tMlr1ZXIa3N5zAyn1p2PvvNZPltW09mY2HliYi/XoxMvNu4ZvdF5B3q1yyrCM+889mSbde3b9kL46k38SDOonchmgHNxWO+EUNYEIxEVEd1bxRfRx4bSC6ffAXAPEkdx/c2x79WjWCIAh47sfDNb6Gp7sbukbdgZhwf5y4kl/bKmvYes0sAHjk6/2a99qBydnsAvyRkoWn+0SjnlL6MfjM7VXGX/rlKLLzS5GaW4TDaTeweFycXllH7K4xtVSFua0w2j9TbLkhIiK7qO9d/XDWfqApFAoMax+K6Eb1anV+dzcFPN3d8PvU3ogysu6TIzqVWR2Mabc6DPlsFz7bdhbztpwxeY6rBZWBDQD8ZWDZC0fqlqpirRHxCrbcEBGRvXl7uKNhfS+UlKslhz83qKeUOMpyCoXCIbtfDNl19qros1otYNfZq/hh3yXNtsNpN02eR7u1orRCDbVa0OtOc8T7Yq1FRpO1cnPYckNERHbh5qbA3lcHIumNQZJDqxv5KTVz5FQ5/Z9hZp07xF8cGDliC4UhE749IPqsUguY8O0BbD2ZrdlmTneSboLzb8eu6JVxxPtirbkeX/rlmOZ9RR2a0IfBDRFRHefl4QZvT3eD+w+9OVj0Warso931E5B1/1DXnRG4Lpm9Vn8IuTktEbplpIaqG4ptCkrKkX692LwKWtnxjHy9mZ5r2/LClhsiInJoyx+/U/N+SLsQfHBvB70yb45qJ/ps7rIIjqhMIjAz52FtTmuFoZabO9/fhj7/3S5bgPP93ouiz0t3/lur8+nOEO3IGNwQEbmgu9oEa957eYgfBeN7NMWhNwbh7thw0fZSreDmo/v1g6G6xpzuJN34R2rtquIyleSxJeWV90t7VmV7StFpZfpOJ9gRBAGHLl3HDQMT/p3OEo+OMxQMplzOwwurk3H5hjxBnBQGN0RELkAqv7Rh/cqcmuHtw/TKNqivn4is3XIz9k7rzKMjp7PZhVifnKH5rFYLKK0QByq6t81Qmm7SxesGr1MVEAmCgCIjy0toU6sFvPXrcaw9fNms8jWx48xV3L8kEZ3/s1Vy/yNfidfuMjRaavTCPfj1yBU89V2Sw3RdMbghInIBuknFAPDn9D5Y9XR3jOgQatY5vpnYFV4ebvj4gY6S+3u3aCg567Ajm77mCB5cuhfncwrxxHcH0fuj7XrdOeb4ZMtZk2VeSTiGmLf/xLHLN02W3XIyGysSL2HG/44aLZdbWIqEQ5dRUq7feqQb0Oo2VP15Ikvz/srNWwAq84Sq8opuFIsnLTTV0nUmuwD3LHKMNagY3BARuYD3xlR2I00f1FKzrUF9JXq2aCjZ1SKlf+tgnHx3KB7sGgEAGNRWvCzDwkc7S47Y0ja8vX4gdYevp6zdXAcv3sCsn49i59mruFpQirc3nNDsu6bTZXOzuBzPrTykd47EC9dEI7G0VcV7/0uqbIVZvN107su1ItOrqgOVK6PP/PkoPtp8Wm/fr0euYM3BNK0thoOT5LSbUKsFxL67BaO+2IPd567qldHNuckrLhcNFQcqE5kdAYMbIiIXMKx9KFLeGYLpg1qZLNs5MtDgPg+t4OXTseI1rAJ9vRDg42n03FJ//D/dpxk8tMYu39MpXL+QjeVoLRBpzJqkdPxxPEty37w/qycFPJBa3U2lGzsaiiW3nczGa+tSsPpAGl5fZ94MzmezCwEAmw3U6ZWE6lFiuvd+9cF0zfuisgpMX3NEk2O0KUX/fNpdToIgYMj8nbh38V6z6mlvDG6IiFyEn7fxwGPbjH6Y92As7ok1b1Vxf4nzfTlef3mCP17oo3kvlbfxbN9molFJ88d2Muv61lRuhVyRMpUa28/koLisAnu0Wj5eXHNULzkXAErKVbhwtVDz+ekVSVi1Pw2vSgxbr3I8Iw9Rr27EisSLou3mTNpn7BveKlNhw9HqOXyk5gCq+jcSBAGPLduP7Hzp1qXiMvPyimyJwQ0REQEAWgTXxwNxTWq0oGWz28s8xEYEYkJ8U4RrzZbcJtQP93VpjEe7R0rm5Hi4u4kmyzO3m8yarhaY1w1kTGpuEZ5YfhBTViXr7Zv0g7grK6egBG3e3IwBn+zE3n9zzb7GqC8qc1re+vWEaLubW2WAYsj/DqbjuoFRUUBly402qdXZq1purhWV4Z/zhkeAvbn+hMF99sLlF4iIqMZ2v3wXTmbmY0i76vybOfe0x5x72uPklXz4eVeuUv7pQ50AAE9/f1B0fPPbQVGFkQkCY5sE4Ohl/cnzHNXfp3PgpZN7dElrrhuFAvhs6znN53WHM9CzecNaXdNNocBvR/VnT67ycsIxg/sA/cAoQWKUVlWr24Yjhq9TdewnD8UaLWNrbLkhIqIaiwjyxdCYUMnWlnbh/ogIEi+2GdskUPTZ/3aOTnigj2h7v1aNAAAD2gRj2eN34vGeUejTsiE6NA6wYu1tZ/MJcc6Kdi+PAgqUao1u2nYqG9lm5vwY4qZQoNzAhINnswtMHl9QYrorSaUWkHerHIu2nzdZVi3zkHC23BARkd0807cZvD3d8f6mUwCg6aYa3C4Es4a0QmxEIABg3oOx2HIyC/d2bgxfLw+8c3eM5hz3L9mLQ5du6J3bXMF+SuRYoRuqxhQQTZhzo7gcQz7bVatTuikM592Yc25zJuD7ctcFvQVJDVEJAtwMzgpke2y5ISIiu/H2dMczfZtpPleNklIoFJgyoCX6tKxssWnkp8S47k3h66X/N7h2suvyJ+7U22/MsoldseKpbjWputVsPJap11KTd6vcQOlqP+gkEWtzUyhqFUpsO5Vjsoy5gQ0g/1INDG6IiEg2Hu6WP5K1H5t3tQ7GhPimBssqFMDUAS00n90UCvh6yt9pYSwhV9dbv1YOC39TJ4lYO0A6l1NocIi5HAx1kdkLgxsiIpKNr5fh1cwN0R2l/O7dMdj10l1o5Ke/ZMTPz8Zj5pDWms931POCt6flj76WwfUtPsZaViRektz+381nJLc7AhVbboiIyNXMuScGTRv44o2R7UwX1qH72FQoFIhs4IsH4ppoto3oEIq1z/dE16ggAMAnD8bihYEt0SkiEEoPywMquROZo17dqLdNd0RT1YR+jkDulhv52+aIiMjlTIiPwoT4qJodbGCNI+1emcXjxJMJ3q8V+Chr0HLTvVkQ1motsglUzulzNP2mxeeylWV7UuWugobcC2iy5YaIiOqUqkAlJtxftN3U0g9VlB7Sj75+rRppup8e7xml2f7Z2FiM6BCmV/7t0Za3OjmakR31v5c1vJpgeJZle2DLDRER1SmPdW+KViF+aK/TVTQ+vikSL1zD4HYhBo6spD0nz3dP3AlBANqE+SEswAelFSr8fjQTd7UJxkNdI3CjuAy9WuhPsPdc/+ZmLXng6CbGR2HjsUyrn3enBSOrbIEtN0REVKe4uSnQo1kD1FeK/z739fLAd090w7juhkdPVdn76gBsm9EP/VsH4642wQgLqJxEUOnhjvvjmiConhfahfuLApt7O1euudXIT4npg1qifbg/2oVVth4F1fNC2zB/vevMHGx6oVJr+2ys+bMD+3iK848m9Wtu7erIgi03RETkcnRnRDbHR/d3xBO9otA+PECz/tbGab2hUgsQAHi6u6Hfx9tx6VrlhHixTQJwd6dwfLL1rDWrbpKHmxtS545A+7f/RJGR9aaiGvgi0FfclVdfaXmydZXF47rg+R8P1/h4a2LLDRERkRm8PNzQsUmgaGFRhUIBD3c3eN5eS2r7zP7oEhkIAPj+yW6IDPLFw3dGAKhs3ZGy9DH9ldRrw9vTHQqFQjQ0PjLIV7OkRZX37+2gV6cWEkPe/bzNawcx1R1oT7IHN4sXL0Z0dDS8vb0RFxeH3bt3Gy2/c+dOxMXFwdvbG82aNcPSpUvtVFMiIiLj3NwUWPt8L1z8cCQCfb2gUCjw4f0dcfHDkXh1eBvJY4bGhOA/98Tg18m98Of0vgCAQF9PrHyqu17Zo28NMVmHqiBGOydo5pBWWDyui6ict6cb6ml17fVt1QhDY0IxW6eew9uHat5/M6Gr5DVfG9EGnu5u+GtmP4T6e+OtUfImW8sa3KxZswbTp0/H66+/juTkZPTp0wfDhw9HWlqaZPnU1FSMGDECffr0QXJyMl577TVMmzYNCQkJdq45ERGRZUZ2CEOzRvUwIb4pwgK8NdsVCgXGx0chNiIQrUP9kDh7APa8MgC9W4oTmf+vbzME+OqPCGvfWJzr4+VRtaSF+Nr1lB5Ieac6OKqa72fnS/3x+oi2WPpYFygUCjzbr7lmzS8AeHFwK3SLCsIXj3TGoHYhaNawnuh6of7e+L++lbk6zRvVx77XBuLJ3tGW3BqrUwiCgQkD7KB79+7o0qULlixZotnWtm1bjBkzBnPnztUr/8orr2DDhg04deqUZtukSZNw9OhRJCYmmnXN/Px8BAQEIC8vD/7++slfREREtnYmqwBzfj+BGYNbIa5pkMFyC/46h0+3nsXTvaPxxu3WkLRrxXjpl6PYn3odnSMD8fnYzuj78XYAwLF3hsDfuzIA2nw8E5NWHsYj3SIx974OACrX5YqevQkAcPD1QZKzOleV+37vRXRpegc66qzkXlhagYu5RVAogA1Hr+Cx7k31Vn+3BUue37IFN2VlZfD19cXPP/+Me++9V7P9hRdewJEjR7Bz5069Y/r27YvOnTvj888/12xbt24dHnroIRQXF8PTUz+iLS0tRWlp9eqv+fn5iIiIYHBDRER1Qm5hKQJ9POHhXt3ZUq5SY9fZq+gaFWR0fp+cghI0qq8UDX//53wubpWpMMiBcmTMYUlwI1u3VG5uLlQqFUJCxDc3JCQEWVlZksdkZWVJlq+oqEBubq7kMXPnzkVAQIDmFRERYZ0vQEREZAcN6ytFgQ1QOTJrYNsQkxMXBvt5iwIbAOjVomGdC2wsJXtCse5NFwRBb5up8lLbq8yePRt5eXmaV3p6ei1rTERERI5MtnluGjZsCHd3d71WmpycHL3WmSqhoaGS5T08PNCgQQPJY5RKJZRK6T5FIiIicj6ytdx4eXkhLi4OW7duFW3funUrevbsKXlMfHy8XvktW7aga9eukvk2RERE5Hpk7ZaaMWMGvvnmG3z77bc4deoUXnzxRaSlpWHSpEkAKruUJkyYoCk/adIkXLp0CTNmzMCpU6fw7bffYtmyZZg1a5ZcX4GIiIgcjKzLL4wdOxbXrl3DnDlzkJmZifbt22PTpk1o2rRyXZDMzEzRnDfR0dHYtGkTXnzxRSxatAjh4eFYsGAB7r//frm+AhERETkYWee5kQPnuSEiIqp76sRQcCIiIiJbYHBDREREToXBDRERETkVBjdERETkVBjcEBERkVNhcENEREROhcENERERORVZJ/GTQ9W0Pvn5+TLXhIiIiMxV9dw2Z3o+lwtuCgoKAAAREREy14SIiIgsVVBQgICAAKNlXG6GYrVajStXrsDPzw8KhcKq587Pz0dERATS09M5+7EN8T7bB++z/fBe2wfvs33Y6j4LgoCCggKEh4fDzc14Vo3Ltdy4ubmhSZMmNr2Gv78//+PYAe+zffA+2w/vtX3wPtuHLe6zqRabKkwoJiIiIqfC4IaIiIicCoMbK1IqlXj77behVCrlropT4322D95n++G9tg/eZ/twhPvscgnFRERE5NzYckNEREROhcENERERORUGN0RERORUGNwQERGRU2FwYyWLFy9GdHQ0vL29ERcXh927d8tdpTrlnXfegUKhEL1CQ0M1+wVBwDvvvIPw8HD4+Pigf//+OHHihOgcpaWlmDp1Kho2bIh69erh7rvvxuXLl+39VRzKrl27MHr0aISHh0OhUGD9+vWi/da6rzdu3MD48eMREBCAgIAAjB8/Hjdv3rTxt3Mcpu7z448/rvfz3aNHD1EZ3mfT5s6dizvvvBN+fn4IDg7GmDFjcObMGVEZ/kzXnjn32dF/phncWMGaNWswffp0vP7660hOTkafPn0wfPhwpKWlyV21OiUmJgaZmZmaV0pKimbff//7X3z66adYuHAhDh48iNDQUAwePFizVhgATJ8+HevWrcPq1auxZ88eFBYWYtSoUVCpVHJ8HYdQVFSE2NhYLFy4UHK/te7ro48+iiNHjmDz5s3YvHkzjhw5gvHjx9v8+zkKU/cZAIYNGyb6+d60aZNoP++zaTt37sTkyZOxb98+bN26FRUVFRgyZAiKioo0ZfgzXXvm3GfAwX+mBaq1bt26CZMmTRJta9OmjfDqq6/KVKO65+233xZiY2Ml96nVaiE0NFT48MMPNdtKSkqEgIAAYenSpYIgCMLNmzcFT09PYfXq1ZoyGRkZgpubm7B582ab1r2uACCsW7dO89la9/XkyZMCAGHfvn2aMomJiQIA4fTp0zb+Vo5H9z4LgiBMnDhRuOeeewwew/tcMzk5OQIAYefOnYIg8GfaVnTvsyA4/s80W25qqaysDIcOHcKQIUNE24cMGYK9e/fKVKu66dy5cwgPD0d0dDQefvhhXLhwAQCQmpqKrKws0T1WKpXo16+f5h4fOnQI5eXlojLh4eFo3749/x0MsNZ9TUxMREBAALp3764p06NHDwQEBPDea9mxYweCg4PRqlUrPPPMM8jJydHs432umby8PABAUFAQAP5M24rufa7iyD/TDG5qKTc3FyqVCiEhIaLtISEhyMrKkqlWdU/37t2xYsUK/Pnnn/j666+RlZWFnj174tq1a5r7aOweZ2VlwcvLC3fccYfBMiRmrfualZWF4OBgvfMHBwfz3t82fPhw/Pjjj/j777/xySef4ODBgxgwYABKS0sB8D7XhCAImDFjBnr37o327dsD4M+0LUjdZ8Dxf6ZdblVwW1EoFKLPgiDobSPDhg8frnnfoUMHxMfHo3nz5vj+++81SWo1ucf8dzDNGvdVqjzvfbWxY8dq3rdv3x5du3ZF06ZNsXHjRtx3330Gj+N9NmzKlCk4duwY9uzZo7ePP9PWY+g+O/rPNFtuaqlhw4Zwd3fXizJzcnL0/nog89WrVw8dOnTAuXPnNKOmjN3j0NBQlJWV4caNGwbLkJi17mtoaCiys7P1zn/16lXeewPCwsLQtGlTnDt3DgDvs6WmTp2KDRs2YPv27WjSpIlmO3+mrcvQfZbiaD/TDG5qycvLC3Fxcdi6dato+9atW9GzZ0+ZalX3lZaW4tSpUwgLC0N0dDRCQ0NF97isrAw7d+7U3OO4uDh4enqKymRmZuL48eP8dzDAWvc1Pj4eeXl5OHDggKbM/v37kZeXx3tvwLVr15Ceno6wsDAAvM/mEgQBU6ZMwdq1a/H3338jOjpatJ8/09Zh6j5Lcbif6VqlI5MgCIKwevVqwdPTU1i2bJlw8uRJYfr06UK9evWEixcvyl21OmPmzJnCjh07hAsXLgj79u0TRo0aJfj5+Wnu4YcffigEBAQIa9euFVJSUoRHHnlECAsLE/Lz8zXnmDRpktCkSRNh27ZtwuHDh4UBAwYIsbGxQkVFhVxfS3YFBQVCcnKykJycLAAQPv30UyE5OVm4dOmSIAjWu6/Dhg0TOnbsKCQmJgqJiYlChw4dhFGjRtn9+8rF2H0uKCgQZs6cKezdu1dITU0Vtm/fLsTHxwuNGzfmfbbQc889JwQEBAg7duwQMjMzNa/i4mJNGf5M156p+1wXfqYZ3FjJokWLhKZNmwpeXl5Cly5dREPmyLSxY8cKYWFhgqenpxAeHi7cd999wokTJzT71Wq18PbbbwuhoaGCUqkU+vbtK6SkpIjOcevWLWHKlClCUFCQ4OPjI4waNUpIS0uz91dxKNu3bxcA6L0mTpwoCIL17uu1a9eEcePGCX5+foKfn58wbtw44caNG3b6lvIzdp+Li4uFIUOGCI0aNRI8PT2FyMhIYeLEiXr3kPfZNKl7DEBYvny5pgx/pmvP1H2uCz/TittfhIiIiMgpMOeGiIiInAqDGyIiInIqDG6IiIjIqTC4ISIiIqfC4IaIiIicCoMbIiIicioMboiIiMipMLghIiIip8LghoiIiJwKgxsiIiJyKgxuiIiIyKkwuCEiIiKn8v/3lykS0H5Z7QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(loss_history)\n",
    "plt.ylabel('train loss')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "436f3944",
   "metadata": {},
   "source": [
    "## 模型预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "2c99ea7d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3333333333333333\n"
     ]
    }
   ],
   "source": [
    "model.eval()\n",
    "translation_results = []\n",
    "\n",
    "correct = 0\n",
    "error = 0\n",
    "\n",
    "for enc_inputs, dec_inputs, dec_outputs in test_loader:\n",
    "    '''\n",
    "    enc_inputs: [batch_size, src_len]\n",
    "    dec_inputs: [batch_size, tgt_len]\n",
    "    dec_outputs: [batch_size, tgt_len]\n",
    "    '''\n",
    "    # enc_inputs, dec_inputs, dec_outputs = enc_inputs.to(device), dec_inputs.to(device), dec_outputs.to(device)\n",
    "    # outputs: [batch_size * tgt_len, tgt_vocab_size]\n",
    "    outputs, enc_self_attns, dec_self_attns, dec_enc_attns = model(enc_inputs, dec_inputs)\n",
    "    # pred形状为 (seq_len, batch_size, vocab_size) = (1, 1, vocab_size)\n",
    "    # dec_outputs, dec_self_attns, dec_enc_attns = model.decoder(dec_inputs, enc_inputs, enc_output)\n",
    "    \n",
    "    outputs = outputs.squeeze()\n",
    "    \n",
    "    pred_seq = []\n",
    "    for output in outputs:\n",
    "        next_token_index = output.argmax().item()\n",
    "        if next_token_index == tgt_vocab['<eos>']:\n",
    "            break\n",
    "        pred_seq.append(next_token_index)\n",
    "    \n",
    "    pred_seq = tgt_vocab[pred_seq]\n",
    "    tgt_seq = dec_outputs.squeeze().tolist()\n",
    "    \n",
    "    # 需要注意在<eos>之前截断\n",
    "    if tgt_vocab['<eos>'] in tgt_seq:\n",
    "        eos_idx = tgt_seq.index(tgt_vocab['<eos>'])\n",
    "        tgt_seq = tgt_vocab[tgt_seq[:eos_idx]]\n",
    "    else:\n",
    "        tgt_seq = tgt_vocab[tgt_seq]\n",
    "    translation_results.append((' '.join(tgt_seq), ' '.join(pred_seq)))\n",
    "    \n",
    "    for i in range(len(tgt_seq)):\n",
    "        if i >= len(pred_seq) or pred_seq[i] != tgt_seq[i]:\n",
    "            error += 1\n",
    "        else:\n",
    "            correct += 1\n",
    "    \n",
    "print(correct/(correct+error))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "87a19f24",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('h x n y e k', 'h y y y k'),\n",
       " ('y l z k i t', 't i t j i t y'),\n",
       " ('t s x e e v', 's s v e e v'),\n",
       " ('e g a m t h', 'f i h h h'),\n",
       " ('d b v t l r', 'e r l l r r'),\n",
       " ('d b e p m m', 'b m e m m'),\n",
       " ('e j r d f w', 'g r w w g'),\n",
       " ('p n p v w j', 'v n j v w j'),\n",
       " ('m q c v g w', 'n g g v w w'),\n",
       " ('q b j e e y', 'q y y e y y'),\n",
       " ('i x j g h z', 'g g g g h g'),\n",
       " ('m z d t r f', 'z d d f r'),\n",
       " ('o y g a c g', 'a a g g c g'),\n",
       " ('m c w z c s', 's s s s c'),\n",
       " ('h b f r z g', 'z z r g g g'),\n",
       " ('h c i o l m', 'c c o c m m'),\n",
       " ('r z e e u t', 'u e e t t t r'),\n",
       " ('g d h d q y', 'y d y y y'),\n",
       " ('r l m s u r', 'f l u s r r r'),\n",
       " ('a v q s k x', 's s s s x'),\n",
       " ('m c g a e u', 'g g a a u u'),\n",
       " ('k y y o b u', 'k o o o b u'),\n",
       " ('r n n i o s', 'o n o o s s s'),\n",
       " ('x x k y n u', 'h n n n n u'),\n",
       " ('l v q a p i', 'q i a a p i v'),\n",
       " ('v r m h w s', 'h r h h s s'),\n",
       " ('x m i o z t', 'i i i q q m'),\n",
       " ('o c y y e j', 'y y y e e j'),\n",
       " ('k w j w a d', 'w w j n n n'),\n",
       " ('m u c l x u', 'l u x l u'),\n",
       " ('y u p g r k', 'u u s s k'),\n",
       " ('q i e t j q', 'e j q q q q l'),\n",
       " ('f q s k n a', 'f a a a a a'),\n",
       " ('i i m o l i', 'l l l l i i'),\n",
       " ('t e y i i j', 'y i l i j'),\n",
       " ('w j p o p f', 'p f s f d f'),\n",
       " ('l r m c z c', 'm c c c z r'),\n",
       " ('a w e i x w', 'w w x i v a'),\n",
       " ('u h q e n v', 'q n e n v v h'),\n",
       " ('c z y l g l', 'y g l l l l l'),\n",
       " ('z q r y x b', 'r r b x q b q'),\n",
       " ('g m g b o w', 'g o w o w'),\n",
       " ('m h y p l y', 'h h y l l s'),\n",
       " ('h r d n q m', 'q r q m r m r'),\n",
       " ('m u c i q u', 'u u c i k m m'),\n",
       " ('q q n u q u', 'q q u u q q'),\n",
       " ('u q t a r b', 'r r b z r'),\n",
       " ('o v j f e w', 'f f p e e f'),\n",
       " ('o r y p y k', 'y y k p v'),\n",
       " ('i c b s j u', 'u u y j u i'),\n",
       " ('n e u a j o', 'j j j o u e'),\n",
       " ('o q n x e x', 'q q x x k'),\n",
       " ('n g u b s z', 'b g s s s z'),\n",
       " ('n f e x a r', 'n q r x a e'),\n",
       " ('f m b y p w', 'w m b p p m'),\n",
       " ('e k m f w n', 'w f n w n n'),\n",
       " ('y y m b b z', 'm m b b z z y'),\n",
       " ('i j v g k l', 's y s s s'),\n",
       " ('a w e g p m', 'm e p g p m'),\n",
       " ('k y q v a o', 'y a a v a o y'),\n",
       " ('w s f v d u', 'w r d v d r'),\n",
       " ('a r h k q v', 'r h k k q v'),\n",
       " ('t x x k s g', 's s s s s g'),\n",
       " ('e v q f e g', 'q q q f g g'),\n",
       " ('t k l s m h', 's k s m h h'),\n",
       " ('x u b e d t', 'b b d e d t'),\n",
       " ('r f i d s z', 'z z z d s z'),\n",
       " ('m g f y s g', 's s s s s g'),\n",
       " ('t f d u d e', 'u u u z z z'),\n",
       " ('f x y g b c', 'c g y g c c'),\n",
       " ('p u z n b u', 'b u n n w u'),\n",
       " ('k y e j h m', 'h m h h m m'),\n",
       " ('b r u u r b', 'r u u n n w n'),\n",
       " ('y j r h e u', 'y u u l u u r'),\n",
       " ('o q j k x c', 'q c c c x'),\n",
       " ('w s d t d r', 'd d d r r r'),\n",
       " ('e w x i s v', 'i i s i v v'),\n",
       " ('d z s q d g', 'b s s s b b'),\n",
       " ('a n x y w g', 'r w l l l l'),\n",
       " ('e a f y k c', 'c c c c e c'),\n",
       " ('h k h u d m', 'h s s s s f'),\n",
       " ('u z r d t k', 'd d d y y'),\n",
       " ('f l v z i j', 'v l j s j j'),\n",
       " ('g e n w h l', 'h e h h h l v'),\n",
       " ('l u l c a p', 'u u a p p h c'),\n",
       " ('m a q g v g', 'v g g g v c'),\n",
       " ('p t p t r r', 'w k r r r'),\n",
       " ('i w e q g o', 'i g g g o o'),\n",
       " ('w u n o v x', 'r n v o v z'),\n",
       " ('y s q n i i', 'i i i i i c s'),\n",
       " ('u g h j l d', 'd d j l v'),\n",
       " ('m l b v v u', 'b l u v v u'),\n",
       " ('y k q s y o', 's s s s y'),\n",
       " ('e f d v l a', 'a a a l l g'),\n",
       " ('t m r a z z', 'z r z z z z'),\n",
       " ('g z k f p r', 'g k k r g r'),\n",
       " ('b d k e m e', 'e e e e e e t'),\n",
       " ('q q h x f t', 'q m m x y q'),\n",
       " ('i p q w d p', 'p p q w d p'),\n",
       " ('u s b z k o', 's s z k k s'),\n",
       " ('k p q z y d', 'k d s z d d s'),\n",
       " ('n w i s m x', 's i s s x x'),\n",
       " ('n i x u l k', 'u g g k k k'),\n",
       " ('i j q e l w', 'i r q w l w'),\n",
       " ('y e p w f c', 'y f c c y'),\n",
       " ('i i r m q i', 'q m m q q r q'),\n",
       " ('d t e z l k', 'e e z z l'),\n",
       " ('e l j z p b', 'b q y z p'),\n",
       " ('o s y o a h', 's s y a h h'),\n",
       " ('z k q q q c', 'u q q q c c q'),\n",
       " ('f d c n n e', 'f e c e e e'),\n",
       " ('n e d m i s', 'd i i m i s'),\n",
       " ('j o l b e n', 'l o a b e a'),\n",
       " ('p r g t k d', 'd d d t k d r'),\n",
       " ('v d h e f u', 'v d u u u h'),\n",
       " ('q j l g t r', 'r r r g a r'),\n",
       " ('u v j a k g', 'a a k a g g'),\n",
       " ('w j z w a o', 'z z a a o'),\n",
       " ('o h l e d a', 'l d l d d'),\n",
       " ('v r n q e b', 'b b b q b v'),\n",
       " ('g n z r o b', 'b r z r o b'),\n",
       " ('b g q u u k', 'b u u u k k'),\n",
       " ('e b q j d i', 'i q i i i'),\n",
       " ('o h u j b a', 'j u a a b a'),\n",
       " ('i v i y v t', 'i y y t t t v'),\n",
       " ('a m x l l p', 'm m r r l p'),\n",
       " ('r a c g m z', 'r a g z z z'),\n",
       " ('q c y v v d', 'n y v d d d'),\n",
       " ('d e j m j v', 'j j j w v v'),\n",
       " ('h j g q p o', 'q q q q o o'),\n",
       " ('x g t i r k', 'x i r i r'),\n",
       " ('f g e c l s', 'e s s s s r'),\n",
       " ('c x h l s j', 'l s s s j j'),\n",
       " ('a k x r h d', 'z z z d d e'),\n",
       " ('h q f r a g', 'f f a g a g'),\n",
       " ('f u a a o h', 'f u a h h h'),\n",
       " ('p t j q u a', 'a a j a a a'),\n",
       " ('t i y c t h', 's s s s h h'),\n",
       " ('g h d v q n', 'g h v v o g'),\n",
       " ('p a f e m g', 'p e e g g g'),\n",
       " ('o y w p q x', 'p p q q q w'),\n",
       " ('y x f t o d', 'y d f o o d'),\n",
       " ('f l x d f l', 'z l d l l l'),\n",
       " ('o m n v o w', 'w o w w w z'),\n",
       " ('y n w g p t', 'y p g g t t y'),\n",
       " ('d y s t k i', 's s s i k s'),\n",
       " ('h h m p w h', 'h m m q q h h'),\n",
       " ('p q h d d d', 'q q d d d d'),\n",
       " ('c h t l j f', 'h l l l j i'),\n",
       " ('x e d t j d', 'e e d l d s'),\n",
       " ('w i i e f n', 'i i i f y'),\n",
       " ('b l k d l s', 'k k s r s s'),\n",
       " ('u s p d u d', 'd d d d u v'),\n",
       " ('e a g s o c', 'a a s s c c'),\n",
       " ('c s f k l n', 'q s f g l'),\n",
       " ('l f g l u b', 'l u g l u b'),\n",
       " ('k e s m e v', 'q s s v r l'),\n",
       " ('q r q j r a', 'r r j a r'),\n",
       " ('v r d j w a', 'a a j a w'),\n",
       " ('y k y v k r', 'k k r d r r l'),\n",
       " ('l h e i x a', 'x i a i v a'),\n",
       " ('o r g u b h', 'h i b u b h'),\n",
       " ('h n c l v g', 'g c l g v h'),\n",
       " ('f d u j r f', 'f u u j r'),\n",
       " ('p a h i p a', 'p i h i a a h'),\n",
       " ('u d z t m z', 'd d z u m z u'),\n",
       " ('j i c c n d', 'i i c r r r'),\n",
       " ('u w w q o g', 'q g q q g w'),\n",
       " ('n t i z v t', 'o o z z n'),\n",
       " ('p t o t w b', 'p s o b b b'),\n",
       " ('v n a u o y', 'f a a c y s'),\n",
       " ('u x q k l s', 's s l k s s'),\n",
       " ('o a j u g y', 'o a u u g y'),\n",
       " ('u z t u o t', 't t t o v t'),\n",
       " ('m o b q b f', 'b s b q b q'),\n",
       " ('d e z w b o', 'y o z y y'),\n",
       " ('r m m a d y', 'd d y y d y'),\n",
       " ('t e r d m r', 'e r r l m r'),\n",
       " ('d z z d e x', 'x z u u x x'),\n",
       " ('o p q h f l', 't c c l l l'),\n",
       " ('w k s d k k', 'k k k d k'),\n",
       " ('y i u u k y', 'u a a u y y a'),\n",
       " ('q b j v p o', 'v v v v o o'),\n",
       " ('l p k v r l', 'c c v v l'),\n",
       " ('z b z y o l', 'b o o y l l y'),\n",
       " ('z q u l t y', 'u u l y y y'),\n",
       " ('n v n j v q', 'v v j q q s v'),\n",
       " ('a d b e i y', 'i i i i y y'),\n",
       " ('p i d v i m', 'v i d s s s'),\n",
       " ('p n o a q x', 'n q a q x x'),\n",
       " ('w t p p q g', 'y g g q q l'),\n",
       " ('g o d y s h', 'y d h s s h g'),\n",
       " ('w e n h p a', 'q g n p p i'),\n",
       " ('k y s l s t', 's s s l s t y'),\n",
       " ('d b l i n j', 'l l l j j'),\n",
       " ('w d f y t n', 'n n f t t z'),\n",
       " ('a y t m u p', 'm u m m u j'),\n",
       " ('d r p z z r', 'z r z z r r'),\n",
       " ('c y a j z f', 'j a j z z c'),\n",
       " ('z c p c w c', 'n c c w w')]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "translation_results"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
